2024-12-11 10:11:38,356 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@57bc27f5 2024-12-11 10:11:38,396 main DEBUG Took 0.036405 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-11 10:11:38,397 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-11 10:11:38,397 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-11 10:11:38,401 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-11 10:11:38,404 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 10:11:38,420 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-11 10:11:38,440 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 10:11:38,442 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 10:11:38,443 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 10:11:38,443 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 10:11:38,444 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 10:11:38,444 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 10:11:38,445 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 10:11:38,445 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 10:11:38,446 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 10:11:38,446 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 10:11:38,447 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 10:11:38,448 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 10:11:38,448 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 10:11:38,449 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 10:11:38,449 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 10:11:38,449 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 10:11:38,450 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 10:11:38,450 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 10:11:38,450 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 10:11:38,450 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 10:11:38,453 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 10:11:38,454 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 10:11:38,455 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 10:11:38,455 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 10:11:38,456 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 10:11:38,456 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-11 10:11:38,459 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 10:11:38,461 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-11 10:11:38,468 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-11 10:11:38,476 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-11 10:11:38,477 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-11 10:11:38,478 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-11 10:11:38,493 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-11 10:11:38,497 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-11 10:11:38,505 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-11 10:11:38,505 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-11 10:11:38,506 main DEBUG createAppenders(={Console}) 2024-12-11 10:11:38,507 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@57bc27f5 initialized 2024-12-11 10:11:38,507 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@57bc27f5 2024-12-11 10:11:38,507 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@57bc27f5 OK. 2024-12-11 10:11:38,508 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-11 10:11:38,509 main DEBUG OutputStream closed 2024-12-11 10:11:38,509 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-11 10:11:38,509 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-11 10:11:38,510 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5a56cdac OK 2024-12-11 10:11:38,647 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-11 10:11:38,650 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-11 10:11:38,652 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-11 10:11:38,654 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-11 10:11:38,659 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-11 10:11:38,659 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-11 10:11:38,660 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-11 10:11:38,661 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-11 10:11:38,661 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-11 10:11:38,661 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-11 10:11:38,662 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-11 10:11:38,662 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-11 10:11:38,663 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-11 10:11:38,663 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-11 10:11:38,664 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-11 10:11:38,664 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-11 10:11:38,664 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-11 10:11:38,665 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-11 10:11:38,675 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-11 10:11:38,675 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@1608bcbd) with optional ClassLoader: null 2024-12-11 10:11:38,676 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-11 10:11:38,677 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@1608bcbd] started OK. 2024-12-11T10:11:38,710 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.bucket.TestBucketCachePersister timeout: 13 mins 2024-12-11 10:11:38,714 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-11 10:11:38,714 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-11T10:11:39,145 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1 2024-12-11T10:11:39,191 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestBucketCachePersister#testPrefetchBlockEvictionWhilePrefetchRunning Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=1463, ProcessCount=11, AvailableMemoryMB=4790 2024-12-11T10:11:39,476 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-11T10:11:39,777 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T10:11:39,833 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/testPrefetchBlockEvictionWhilePrefetchRunning 2024-12-11T10:11:39,878 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-11T10:11:39,882 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-11T10:11:39,887 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 200 2024-12-11T10:11:39,899 INFO [Thread-5 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-11T10:11:39,900 WARN [Thread-5 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-11T10:11:39,916 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/testPrefetchBlockEvictionWhilePrefetchRunning, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-11T10:11:39,922 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T10:11:39,929 INFO [Thread-5 {}] bucket.BucketAllocator(343): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-11T10:11:39,930 INFO [Thread-5 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/bucket.persistence is complete. 2024-12-11T10:11:40,010 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-11T10:11:40,161 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 9327a5291c784787abaccbcf2bf4d1e8 2024-12-11T10:11:40,322 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-11T10:11:40,387 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-11T10:11:40,432 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is 235, key is aaaaaaaaaaaaaaaaaaaaaaabababaaba/Pw_:RBzyD/-6268944557396386197/Put/seqid=0 2024-12-11T10:11:40,603 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8, delay=930 ms 2024-12-11T10:11:40,666 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:40,676 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:40,687 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:40,700 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:40,711 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:40,723 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:40,734 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:40,745 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:40,757 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:40,769 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:40,785 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:40,798 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:40,808 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:40,819 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:40,831 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:40,842 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:40,852 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:40,863 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:40,873 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:40,884 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:40,895 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:40,906 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:40,916 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:40,927 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:40,937 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:40,948 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:40,959 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:40,970 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:40,980 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:40,991 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,002 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,013 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,024 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,035 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,045 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,056 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,067 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,077 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,088 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,098 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,109 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,120 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,130 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,141 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,151 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,162 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,172 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,183 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,193 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,204 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,214 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,225 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,235 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,246 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,256 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,267 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,277 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,288 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,298 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,309 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,319 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,330 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,341 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,351 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,362 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,373 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,384 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,394 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,405 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,415 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,426 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,436 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,447 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,457 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,468 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,478 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,489 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,499 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,510 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,520 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,531 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,542 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,547 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_0. 2024-12-11T10:11:41,552 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,563 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,573 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,584 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,594 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,598 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_2120. 2024-12-11T10:11:41,599 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_4212. 2024-12-11T10:11:41,600 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_6400. 2024-12-11T10:11:41,600 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_8515. 2024-12-11T10:11:41,601 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_10671. 2024-12-11T10:11:41,603 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_12909. 2024-12-11T10:11:41,604 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_15092. 2024-12-11T10:11:41,605 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,605 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_17201. 2024-12-11T10:11:41,606 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_19329. 2024-12-11T10:11:41,606 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_21468. 2024-12-11T10:11:41,607 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_23559. 2024-12-11T10:11:41,607 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_25658. 2024-12-11T10:11:41,607 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_27775. 2024-12-11T10:11:41,608 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_29957. 2024-12-11T10:11:41,612 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_32145. 2024-12-11T10:11:41,612 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_34303. 2024-12-11T10:11:41,613 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_36458. 2024-12-11T10:11:41,613 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_38614. 2024-12-11T10:11:41,614 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_40793. 2024-12-11T10:11:41,614 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_42927. 2024-12-11T10:11:41,615 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_45052. 2024-12-11T10:11:41,615 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,615 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_47265. 2024-12-11T10:11:41,616 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_49478. 2024-12-11T10:11:41,616 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_51596. 2024-12-11T10:11:41,617 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_53772. 2024-12-11T10:11:41,617 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_55981. 2024-12-11T10:11:41,618 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_58121. 2024-12-11T10:11:41,618 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_60215. 2024-12-11T10:11:41,619 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_62338. 2024-12-11T10:11:41,621 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_64472. 2024-12-11T10:11:41,622 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_66609. 2024-12-11T10:11:41,622 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_68786. 2024-12-11T10:11:41,623 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_70987. 2024-12-11T10:11:41,623 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_73130. 2024-12-11T10:11:41,623 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_75298. 2024-12-11T10:11:41,624 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_77514. 2024-12-11T10:11:41,624 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_79705. 2024-12-11T10:11:41,625 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_81891. 2024-12-11T10:11:41,625 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,625 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_84038. 2024-12-11T10:11:41,626 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_86192. 2024-12-11T10:11:41,626 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_88315. 2024-12-11T10:11:41,627 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_90450. 2024-12-11T10:11:41,627 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_92572. 2024-12-11T10:11:41,628 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_94658. 2024-12-11T10:11:41,628 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_96748. 2024-12-11T10:11:41,628 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_98873. 2024-12-11T10:11:41,629 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_100988. 2024-12-11T10:11:41,629 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_103185. 2024-12-11T10:11:41,630 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_105314. 2024-12-11T10:11:41,630 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_107488. 2024-12-11T10:11:41,631 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_109612. 2024-12-11T10:11:41,631 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_111761. 2024-12-11T10:11:41,632 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_113872. 2024-12-11T10:11:41,632 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_116024. 2024-12-11T10:11:41,633 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9327a5291c784787abaccbcf2bf4d1e8_117906. 2024-12-11T10:11:41,633 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8, with total blocks 56, and data blocks 55 2024-12-11T10:11:41,634 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-11T10:11:41,635 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8, but only 0 data blocks were cached. Total data blocks for file: 55. Checking for blocks pending cache in cache writer queue. 2024-12-11T10:11:41,636 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,636 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8. Will sleep 100ms and try the verification again. 2024-12-11T10:11:41,646 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,648 DEBUG [Time-limited test {}] bucket.BucketCache(856): removed key 9327a5291c784787abaccbcf2bf4d1e8_2120 from back map with offset lock 519168 in the evict process 2024-12-11T10:11:41,649 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(182): Attempted eviction for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8. Succeeded? true 2024-12-11T10:11:41,659 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,670 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,680 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,691 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,691 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-11T10:11:41,692 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-11T10:11:41,692 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-11T10:11:41,692 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 2, fullycachedFiles size: 0, chunkSize: 10000 2024-12-11T10:11:41,701 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,711 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,722 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,733 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,736 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8, with total blocks 56, and data blocks 55 2024-12-11T10:11:41,737 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2285): iterating over 55 entries in the backing map 2024-12-11T10:11:41,737 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 0 2024-12-11T10:11:41,737 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 4212 2024-12-11T10:11:41,737 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 6400 2024-12-11T10:11:41,737 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 8515 2024-12-11T10:11:41,737 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 10671 2024-12-11T10:11:41,737 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 12909 2024-12-11T10:11:41,738 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 15092 2024-12-11T10:11:41,739 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 17201 2024-12-11T10:11:41,739 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 19329 2024-12-11T10:11:41,739 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 21468 2024-12-11T10:11:41,739 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 23559 2024-12-11T10:11:41,739 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 25658 2024-12-11T10:11:41,740 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 27775 2024-12-11T10:11:41,740 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 29957 2024-12-11T10:11:41,740 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 32145 2024-12-11T10:11:41,741 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 34303 2024-12-11T10:11:41,743 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8 is still running. 2024-12-11T10:11:41,744 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 36458 2024-12-11T10:11:41,744 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 38614 2024-12-11T10:11:41,745 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 40793 2024-12-11T10:11:41,745 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 42927 2024-12-11T10:11:41,745 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 45052 2024-12-11T10:11:41,745 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 47265 2024-12-11T10:11:41,745 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 49478 2024-12-11T10:11:41,746 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 51596 2024-12-11T10:11:41,746 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 53772 2024-12-11T10:11:41,746 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 55981 2024-12-11T10:11:41,746 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 58121 2024-12-11T10:11:41,746 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 60215 2024-12-11T10:11:41,746 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 62338 2024-12-11T10:11:41,747 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 64472 2024-12-11T10:11:41,747 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 66609 2024-12-11T10:11:41,747 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 68786 2024-12-11T10:11:41,747 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 70987 2024-12-11T10:11:41,747 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 73130 2024-12-11T10:11:41,747 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 75298 2024-12-11T10:11:41,748 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 77514 2024-12-11T10:11:41,748 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 79705 2024-12-11T10:11:41,748 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 81891 2024-12-11T10:11:41,748 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 84038 2024-12-11T10:11:41,748 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 86192 2024-12-11T10:11:41,748 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 88315 2024-12-11T10:11:41,749 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 90450 2024-12-11T10:11:41,749 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 92572 2024-12-11T10:11:41,749 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 94658 2024-12-11T10:11:41,749 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 96748 2024-12-11T10:11:41,749 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 98873 2024-12-11T10:11:41,749 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 100988 2024-12-11T10:11:41,750 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 103185 2024-12-11T10:11:41,750 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 105314 2024-12-11T10:11:41,750 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 107488 2024-12-11T10:11:41,750 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 109612 2024-12-11T10:11:41,750 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 111761 2024-12-11T10:11:41,751 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 113872 2024-12-11T10:11:41,751 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 116024 2024-12-11T10:11:41,751 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 9327a5291c784787abaccbcf2bf4d1e8 in the backing map. Acquiring read lock for offset 117906 2024-12-11T10:11:41,751 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8, but only 54 data blocks were cached. Total data blocks for file: 55. Checking for blocks pending cache in cache writer queue. 2024-12-11T10:11:41,751 INFO [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2331): We found only 54 data blocks cached from a total of 55 for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/9327a5291c784787abaccbcf2bf4d1e8, but no blocks pending caching. Maybe cache is full or evictions happened concurrently to cache prefetch. 2024-12-11T10:11:41,753 DEBUG [hfile-prefetch-1733911900605 {}] hfile.PrefetchExecutor(121): Prefetch completed for 9327a5291c784787abaccbcf2bf4d1e8 2024-12-11T10:11:41,754 INFO [Time-limited test {}] bucket.BucketCache(1738): Disabling cache 2024-12-11T10:11:41,755 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-11T10:11:41,755 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-11T10:11:41,756 INFO [Time-limited test {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/bucket.persistence 2024-12-11T10:11:41,757 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-11T10:11:41,758 INFO [Time-limited test {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-11T10:11:41,843 WARN [bucket-cache-persister {}] util.Shell(1106): Interrupted while joining on: Thread[Thread-10,5,FailOnTimeoutGroup] java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Thread.join(Thread.java:1313) ~[?:?] at java.lang.Thread.join(Thread.java:1381) ~[?:?] at org.apache.hadoop.util.Shell.joinThread(Shell.java:1103) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.runCommand(Shell.java:1063) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.getFileSize(PersistentIOEngine.java:94) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.calculateChecksum(PersistentIOEngine.java:71) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.toPB(BucketProtoUtils.java:61) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.serializeAsPB(BucketProtoUtils.java:75) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistChunkedBackingMap(BucketCache.java:1678) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1496) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2024-12-11T10:11:41,860 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 55, fullycachedFiles size: 0 2024-12-11T10:11:41,860 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 55, fullycachedFiles size: 0, file name: bucket.persistence1733911901692 2024-12-11T10:11:41,860 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-11T10:11:41,861 INFO [bucket-cache-persister {}] bucket.BucketCachePersister(62): Finishing cache persister thread. 2024-12-11T10:11:41,861 DEBUG [Time-limited test {}] bucket.BucketCache(1487): Thread Time-limited test started persisting bucket cache to file 2024-12-11T10:11:41,861 DEBUG [Time-limited test {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-11T10:11:41,861 DEBUG [Time-limited test {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 55, fullycachedFiles size: 0, chunkSize: 10000 2024-12-11T10:11:41,877 DEBUG [Time-limited test {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 55, fullycachedFiles size: 0 2024-12-11T10:11:41,878 DEBUG [Time-limited test {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 55, fullycachedFiles size: 0, file name: bucket.persistence1733911901861 2024-12-11T10:11:41,878 DEBUG [Time-limited test {}] bucket.BucketCache(1509): Thread Time-limited test finished persisting bucket cache to file, renaming 2024-12-11T10:11:41,878 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T10:11:41,879 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T10:11:41,883 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T10:11:41,899 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestBucketCachePersister#testPrefetchBlockEvictionWhilePrefetchRunning Thread=20 (was 12) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: hfile-prefetch-1733911900605 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=288 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1442 (was 1463), ProcessCount=11 (was 11), AvailableMemoryMB=4552 (was 4790) 2024-12-11T10:11:41,906 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestBucketCachePersister#testPrefetchPersistenceCrashNegative Thread=20, OpenFileDescriptor=288, MaxFileDescriptor=1048576, SystemLoadAverage=1442, ProcessCount=11, AvailableMemoryMB=4551 2024-12-11T10:11:41,907 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T10:11:41,916 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/testPrefetchPersistenceCrashNegative 2024-12-11T10:11:41,916 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-11T10:11:41,916 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-11T10:11:41,916 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 9223372036854775807 2024-12-11T10:11:41,923 INFO [Thread-13 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-11T10:11:41,924 WARN [Thread-13 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-11T10:11:41,924 INFO [Thread-13 {}] bucket.BucketAllocator(343): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-11T10:11:41,924 INFO [Thread-13 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/bucket.persistence is complete. 2024-12-11T10:11:41,935 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/testPrefetchPersistenceCrashNegative, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-11T10:11:41,936 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T10:11:41,982 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a886b1075ca940ad9a46f5c02cd8609a 2024-12-11T10:11:42,016 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch2/a886b1075ca940ad9a46f5c02cd8609a is 213, key is aaaaaaaaaaaaaaaaaaaaaaabababbabb/NN:_qF6wxdM/-7587313619768436735/Delete/seqid=0 2024-12-11T10:11:42,019 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch2/a886b1075ca940ad9a46f5c02cd8609a, delay=917 ms 2024-12-11T10:11:42,938 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_0. 2024-12-11T10:11:42,939 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_2159. 2024-12-11T10:11:42,940 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_4347. 2024-12-11T10:11:42,941 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_6525. 2024-12-11T10:11:42,941 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_8771. 2024-12-11T10:11:42,942 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_10937. 2024-12-11T10:11:42,942 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_13055. 2024-12-11T10:11:42,943 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_15186. 2024-12-11T10:11:42,944 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_17379. 2024-12-11T10:11:42,944 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_19536. 2024-12-11T10:11:42,945 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_21631. 2024-12-11T10:11:42,945 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_23804. 2024-12-11T10:11:42,946 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_25908. 2024-12-11T10:11:42,946 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_28022. 2024-12-11T10:11:42,947 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_30189. 2024-12-11T10:11:42,947 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_32286. 2024-12-11T10:11:42,948 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_34481. 2024-12-11T10:11:42,948 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_36632. 2024-12-11T10:11:42,948 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_38755. 2024-12-11T10:11:42,949 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_40848. 2024-12-11T10:11:42,949 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_42950. 2024-12-11T10:11:42,950 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_45080. 2024-12-11T10:11:42,950 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_47214. 2024-12-11T10:11:42,951 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_49388. 2024-12-11T10:11:42,951 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_51560. 2024-12-11T10:11:42,951 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_53675. 2024-12-11T10:11:42,952 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_55868. 2024-12-11T10:11:42,952 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_57980. 2024-12-11T10:11:42,953 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_60106. 2024-12-11T10:11:42,954 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_62270. 2024-12-11T10:11:42,954 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_64423. 2024-12-11T10:11:42,955 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_66551. 2024-12-11T10:11:42,955 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_68676. 2024-12-11T10:11:42,956 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_70829. 2024-12-11T10:11:42,956 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_72955. 2024-12-11T10:11:42,956 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_75143. 2024-12-11T10:11:42,957 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_77316. 2024-12-11T10:11:42,957 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_79499. 2024-12-11T10:11:42,958 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_81593. 2024-12-11T10:11:42,958 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_83716. 2024-12-11T10:11:42,958 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_85899. 2024-12-11T10:11:42,959 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_88010. 2024-12-11T10:11:42,959 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_90180. 2024-12-11T10:11:42,960 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_92319. 2024-12-11T10:11:42,960 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_94455. 2024-12-11T10:11:42,960 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_96629. 2024-12-11T10:11:42,961 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_98764. 2024-12-11T10:11:42,961 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_100849. 2024-12-11T10:11:42,961 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_102937. 2024-12-11T10:11:42,962 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_105071. 2024-12-11T10:11:42,962 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_107280. 2024-12-11T10:11:42,962 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_109445. 2024-12-11T10:11:42,963 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_111636. 2024-12-11T10:11:42,963 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_113829. 2024-12-11T10:11:42,964 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_115947. 2024-12-11T10:11:42,964 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a886b1075ca940ad9a46f5c02cd8609a_116473. 2024-12-11T10:11:42,964 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch2/a886b1075ca940ad9a46f5c02cd8609a, with total blocks 56, and data blocks 55 2024-12-11T10:11:42,964 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-11T10:11:42,964 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch2/a886b1075ca940ad9a46f5c02cd8609a, but only 0 data blocks were cached. Total data blocks for file: 55. Checking for blocks pending cache in cache writer queue. 2024-12-11T10:11:42,965 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch2/a886b1075ca940ad9a46f5c02cd8609a. Will sleep 100ms and try the verification again. 2024-12-11T10:11:43,065 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch2/a886b1075ca940ad9a46f5c02cd8609a, with total blocks 56, and data blocks 55 2024-12-11T10:11:43,065 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2285): iterating over 46 entries in the backing map 2024-12-11T10:11:43,065 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 0 2024-12-11T10:11:43,066 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 2159 2024-12-11T10:11:43,066 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 4347 2024-12-11T10:11:43,066 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 6525 2024-12-11T10:11:43,066 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 8771 2024-12-11T10:11:43,066 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 10937 2024-12-11T10:11:43,066 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 13055 2024-12-11T10:11:43,066 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 15186 2024-12-11T10:11:43,066 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 17379 2024-12-11T10:11:43,067 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 19536 2024-12-11T10:11:43,067 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 21631 2024-12-11T10:11:43,067 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 23804 2024-12-11T10:11:43,067 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 25908 2024-12-11T10:11:43,067 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 28022 2024-12-11T10:11:43,067 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 30189 2024-12-11T10:11:43,067 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 32286 2024-12-11T10:11:43,067 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 34481 2024-12-11T10:11:43,067 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 36632 2024-12-11T10:11:43,068 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 38755 2024-12-11T10:11:43,068 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 40848 2024-12-11T10:11:43,068 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 42950 2024-12-11T10:11:43,068 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 45080 2024-12-11T10:11:43,068 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 47214 2024-12-11T10:11:43,068 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 49388 2024-12-11T10:11:43,068 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 51560 2024-12-11T10:11:43,068 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 53675 2024-12-11T10:11:43,068 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 55868 2024-12-11T10:11:43,069 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 57980 2024-12-11T10:11:43,069 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 60106 2024-12-11T10:11:43,069 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 62270 2024-12-11T10:11:43,069 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 64423 2024-12-11T10:11:43,069 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 66551 2024-12-11T10:11:43,069 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 68676 2024-12-11T10:11:43,069 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 70829 2024-12-11T10:11:43,069 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 72955 2024-12-11T10:11:43,069 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 75143 2024-12-11T10:11:43,070 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 77316 2024-12-11T10:11:43,070 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 79499 2024-12-11T10:11:43,070 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 81593 2024-12-11T10:11:43,070 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 83716 2024-12-11T10:11:43,070 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 85899 2024-12-11T10:11:43,070 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 88010 2024-12-11T10:11:43,070 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 90180 2024-12-11T10:11:43,070 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 92319 2024-12-11T10:11:43,070 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 94455 2024-12-11T10:11:43,071 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 96629 2024-12-11T10:11:43,071 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 98764 2024-12-11T10:11:43,071 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 100849 2024-12-11T10:11:43,071 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 102937 2024-12-11T10:11:43,071 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 105071 2024-12-11T10:11:43,071 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 107280 2024-12-11T10:11:43,071 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 109445 2024-12-11T10:11:43,071 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 111636 2024-12-11T10:11:43,071 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 113829 2024-12-11T10:11:43,072 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 115947 2024-12-11T10:11:43,072 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file a886b1075ca940ad9a46f5c02cd8609a in the backing map. Acquiring read lock for offset 116473 2024-12-11T10:11:43,072 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2305): File /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch2/a886b1075ca940ad9a46f5c02cd8609a has now been fully cached. 2024-12-11T10:11:43,073 DEBUG [hfile-prefetch-1733911900605 {}] hfile.PrefetchExecutor(121): Prefetch completed for a886b1075ca940ad9a46f5c02cd8609a 2024-12-11T10:11:44,028 INFO [Time-limited test {}] bucket.BucketCache(1738): Disabling cache 2024-12-11T10:11:44,028 INFO [Time-limited test {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/bucket.persistence 2024-12-11T10:11:44,028 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-11T10:11:44,028 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-11T10:11:44,028 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-11T10:11:44,029 INFO [Time-limited test {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-11T10:11:44,030 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-11T10:11:44,039 DEBUG [Time-limited test {}] bucket.BucketCache(1487): Thread Time-limited test started persisting bucket cache to file 2024-12-11T10:11:44,039 DEBUG [Time-limited test {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-11T10:11:44,040 DEBUG [Time-limited test {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 56, fullycachedFiles size: 1, chunkSize: 10000 2024-12-11T10:11:44,045 DEBUG [Time-limited test {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 56, fullycachedFiles size: 1 2024-12-11T10:11:44,046 DEBUG [Time-limited test {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 56, fullycachedFiles size: 1, file name: bucket.persistence1733911904039 2024-12-11T10:11:44,046 DEBUG [Time-limited test {}] bucket.BucketCache(1509): Thread Time-limited test finished persisting bucket cache to file, renaming 2024-12-11T10:11:44,046 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T10:11:44,047 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T10:11:44,064 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestBucketCachePersister#testPrefetchPersistenceCrashNegative Thread=21 (was 20) Potentially hanging thread: hfile-prefetch-1733911902019 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=289 (was 288) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1442 (was 1442), ProcessCount=11 (was 11), AvailableMemoryMB=4460 (was 4551) 2024-12-11T10:11:44,099 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestBucketCachePersister#testPrefetchListUponBlockEviction Thread=21, OpenFileDescriptor=289, MaxFileDescriptor=1048576, SystemLoadAverage=1442, ProcessCount=11, AvailableMemoryMB=4458 2024-12-11T10:11:44,101 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T10:11:44,123 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/testPrefetchListUponBlockEviction 2024-12-11T10:11:44,123 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-11T10:11:44,124 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-11T10:11:44,124 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 200 2024-12-11T10:11:44,127 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/testPrefetchListUponBlockEviction, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-11T10:11:44,128 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T10:11:44,131 INFO [Thread-18 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-11T10:11:44,131 WARN [Thread-18 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-11T10:11:44,132 INFO [Thread-18 {}] bucket.BucketAllocator(343): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-11T10:11:44,139 INFO [Thread-18 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/bucket.persistence is complete. 2024-12-11T10:11:44,164 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 973cb26647bb4a60b52bbcfff82d97db 2024-12-11T10:11:44,181 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/973cb26647bb4a60b52bbcfff82d97db is 213, key is aaaaaaaaaaaaaaaaaaaaaaaaaabaabab/N:eJiGd/-6298522840219705549/Put/seqid=0 2024-12-11T10:11:44,183 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/973cb26647bb4a60b52bbcfff82d97db, delay=954 ms 2024-12-11T10:11:44,325 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-11T10:11:44,326 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-11T10:11:44,326 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-11T10:11:44,326 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-11T10:11:44,331 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-11T10:11:44,331 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733911904326 2024-12-11T10:11:44,331 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-11T10:11:45,140 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_0. 2024-12-11T10:11:45,141 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_2122. 2024-12-11T10:11:45,142 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_4242. 2024-12-11T10:11:45,143 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_6436. 2024-12-11T10:11:45,143 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_8607. 2024-12-11T10:11:45,144 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_10715. 2024-12-11T10:11:45,144 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_12830. 2024-12-11T10:11:45,145 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_14993. 2024-12-11T10:11:45,145 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_17152. 2024-12-11T10:11:45,145 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_19278. 2024-12-11T10:11:45,146 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_21404. 2024-12-11T10:11:45,146 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_23580. 2024-12-11T10:11:45,146 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_25690. 2024-12-11T10:11:45,147 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_27804. 2024-12-11T10:11:45,147 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_29966. 2024-12-11T10:11:45,147 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_32073. 2024-12-11T10:11:45,148 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_34203. 2024-12-11T10:11:45,148 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_36352. 2024-12-11T10:11:45,148 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_38510. 2024-12-11T10:11:45,149 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_40693. 2024-12-11T10:11:45,149 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_42800. 2024-12-11T10:11:45,149 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_44951. 2024-12-11T10:11:45,149 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_47125. 2024-12-11T10:11:45,150 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_49255. 2024-12-11T10:11:45,150 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_51438. 2024-12-11T10:11:45,150 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_53626. 2024-12-11T10:11:45,150 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_55728. 2024-12-11T10:11:45,151 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_57872. 2024-12-11T10:11:45,151 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_60070. 2024-12-11T10:11:45,151 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_62221. 2024-12-11T10:11:45,152 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_64334. 2024-12-11T10:11:45,152 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_66458. 2024-12-11T10:11:45,152 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_68642. 2024-12-11T10:11:45,152 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_70737. 2024-12-11T10:11:45,153 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_72865. 2024-12-11T10:11:45,153 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_75024. 2024-12-11T10:11:45,153 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_77146. 2024-12-11T10:11:45,154 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_79277. 2024-12-11T10:11:45,154 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_81377. 2024-12-11T10:11:45,154 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_83586. 2024-12-11T10:11:45,155 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_85764. 2024-12-11T10:11:45,155 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_87921. 2024-12-11T10:11:45,155 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_90086. 2024-12-11T10:11:45,156 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_92257. 2024-12-11T10:11:45,156 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_94379. 2024-12-11T10:11:45,156 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_96498. 2024-12-11T10:11:45,156 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_98601. 2024-12-11T10:11:45,157 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_100752. 2024-12-11T10:11:45,157 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_102972. 2024-12-11T10:11:45,158 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_105156. 2024-12-11T10:11:45,158 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_107288. 2024-12-11T10:11:45,158 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_109382. 2024-12-11T10:11:45,158 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_111506. 2024-12-11T10:11:45,159 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_113633. 2024-12-11T10:11:45,159 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_115755. 2024-12-11T10:11:45,159 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_117947. 2024-12-11T10:11:45,160 DEBUG [hfile-prefetch-1733911902019 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 973cb26647bb4a60b52bbcfff82d97db_118894. 2024-12-11T10:11:45,160 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/973cb26647bb4a60b52bbcfff82d97db, with total blocks 57, and data blocks 56 2024-12-11T10:11:45,160 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-11T10:11:45,160 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/973cb26647bb4a60b52bbcfff82d97db, but only 0 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue. 2024-12-11T10:11:45,160 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/973cb26647bb4a60b52bbcfff82d97db. Will sleep 100ms and try the verification again. 2024-12-11T10:11:45,260 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/973cb26647bb4a60b52bbcfff82d97db, with total blocks 57, and data blocks 56 2024-12-11T10:11:45,261 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2285): iterating over 23 entries in the backing map 2024-12-11T10:11:45,261 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 0 2024-12-11T10:11:45,261 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 2122 2024-12-11T10:11:45,261 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 4242 2024-12-11T10:11:45,261 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 8607 2024-12-11T10:11:45,261 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 10715 2024-12-11T10:11:45,262 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 19278 2024-12-11T10:11:45,262 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 23580 2024-12-11T10:11:45,262 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 27804 2024-12-11T10:11:45,262 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 32073 2024-12-11T10:11:45,262 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 34203 2024-12-11T10:11:45,262 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 51438 2024-12-11T10:11:45,262 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 55728 2024-12-11T10:11:45,263 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 70737 2024-12-11T10:11:45,263 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 75024 2024-12-11T10:11:45,263 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 83586 2024-12-11T10:11:45,263 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 85764 2024-12-11T10:11:45,263 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 87921 2024-12-11T10:11:45,263 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 96498 2024-12-11T10:11:45,263 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 98601 2024-12-11T10:11:45,263 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 100752 2024-12-11T10:11:45,263 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 102972 2024-12-11T10:11:45,264 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 105156 2024-12-11T10:11:45,264 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 115755 2024-12-11T10:11:45,264 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/973cb26647bb4a60b52bbcfff82d97db, but only 23 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue. 2024-12-11T10:11:45,264 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/973cb26647bb4a60b52bbcfff82d97db. Will sleep 100ms and try the verification again. 2024-12-11T10:11:45,332 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-11T10:11:45,332 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-11T10:11:45,333 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-11T10:11:45,333 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 57, fullycachedFiles size: 0, chunkSize: 10000 2024-12-11T10:11:45,338 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 57, fullycachedFiles size: 0 2024-12-11T10:11:45,338 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 57, fullycachedFiles size: 0, file name: bucket.persistence1733911905332 2024-12-11T10:11:45,338 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-11T10:11:45,364 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/973cb26647bb4a60b52bbcfff82d97db, with total blocks 57, and data blocks 56 2024-12-11T10:11:45,364 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2285): iterating over 57 entries in the backing map 2024-12-11T10:11:45,364 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 0 2024-12-11T10:11:45,365 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 2122 2024-12-11T10:11:45,365 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 4242 2024-12-11T10:11:45,365 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 6436 2024-12-11T10:11:45,365 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 8607 2024-12-11T10:11:45,365 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 10715 2024-12-11T10:11:45,365 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 12830 2024-12-11T10:11:45,365 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 14993 2024-12-11T10:11:45,365 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 17152 2024-12-11T10:11:45,365 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 19278 2024-12-11T10:11:45,365 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 21404 2024-12-11T10:11:45,365 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 23580 2024-12-11T10:11:45,365 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 25690 2024-12-11T10:11:45,365 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 27804 2024-12-11T10:11:45,365 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 29966 2024-12-11T10:11:45,365 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 32073 2024-12-11T10:11:45,366 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 34203 2024-12-11T10:11:45,366 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 36352 2024-12-11T10:11:45,366 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 38510 2024-12-11T10:11:45,366 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 40693 2024-12-11T10:11:45,366 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 42800 2024-12-11T10:11:45,366 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 44951 2024-12-11T10:11:45,366 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 47125 2024-12-11T10:11:45,366 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 49255 2024-12-11T10:11:45,366 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 51438 2024-12-11T10:11:45,366 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 53626 2024-12-11T10:11:45,366 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 55728 2024-12-11T10:11:45,366 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 57872 2024-12-11T10:11:45,366 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 60070 2024-12-11T10:11:45,367 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 62221 2024-12-11T10:11:45,367 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 64334 2024-12-11T10:11:45,367 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 66458 2024-12-11T10:11:45,367 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 68642 2024-12-11T10:11:45,367 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 70737 2024-12-11T10:11:45,367 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 72865 2024-12-11T10:11:45,367 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 75024 2024-12-11T10:11:45,367 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 77146 2024-12-11T10:11:45,367 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 79277 2024-12-11T10:11:45,367 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 81377 2024-12-11T10:11:45,367 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 83586 2024-12-11T10:11:45,367 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 85764 2024-12-11T10:11:45,367 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 87921 2024-12-11T10:11:45,367 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 90086 2024-12-11T10:11:45,367 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 92257 2024-12-11T10:11:45,367 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 94379 2024-12-11T10:11:45,367 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 96498 2024-12-11T10:11:45,368 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 98601 2024-12-11T10:11:45,368 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 100752 2024-12-11T10:11:45,368 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 102972 2024-12-11T10:11:45,368 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 105156 2024-12-11T10:11:45,368 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 107288 2024-12-11T10:11:45,368 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 109382 2024-12-11T10:11:45,368 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 111506 2024-12-11T10:11:45,368 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 113633 2024-12-11T10:11:45,368 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 115755 2024-12-11T10:11:45,368 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 117947 2024-12-11T10:11:45,368 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2293): found block for file 973cb26647bb4a60b52bbcfff82d97db in the backing map. Acquiring read lock for offset 118894 2024-12-11T10:11:45,368 DEBUG [hfile-prefetch-1733911902019 {}] bucket.BucketCache(2305): File /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch3/973cb26647bb4a60b52bbcfff82d97db has now been fully cached. 2024-12-11T10:11:45,369 DEBUG [hfile-prefetch-1733911902019 {}] hfile.PrefetchExecutor(121): Prefetch completed for 973cb26647bb4a60b52bbcfff82d97db 2024-12-11T10:11:46,191 DEBUG [Time-limited test {}] bucket.BucketCache(856): removed key 973cb26647bb4a60b52bbcfff82d97db_77146 from back map with offset lock 384000 in the evict process 2024-12-11T10:11:46,192 DEBUG [Time-limited test {}] bucket.BucketCache(768): Removing file 973cb26647bb4a60b52bbcfff82d97db for region 6e6df6e1-408a-5bad-4c56-dd8c52038dc1 2024-12-11T10:11:46,192 INFO [Time-limited test {}] bucket.BucketCache(1738): Disabling cache 2024-12-11T10:11:46,192 INFO [Time-limited test {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/bucket.persistence 2024-12-11T10:11:46,193 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-11T10:11:46,193 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-11T10:11:46,193 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-11T10:11:46,193 INFO [Time-limited test {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-11T10:11:46,193 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-11T10:11:46,203 DEBUG [Time-limited test {}] bucket.BucketCache(1487): Thread Time-limited test started persisting bucket cache to file 2024-12-11T10:11:46,204 DEBUG [Time-limited test {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-11T10:11:46,204 DEBUG [Time-limited test {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 56, fullycachedFiles size: 0, chunkSize: 10000 2024-12-11T10:11:46,217 DEBUG [Time-limited test {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 56, fullycachedFiles size: 0 2024-12-11T10:11:46,217 DEBUG [Time-limited test {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 56, fullycachedFiles size: 0, file name: bucket.persistence1733911906203 2024-12-11T10:11:46,217 DEBUG [Time-limited test {}] bucket.BucketCache(1509): Thread Time-limited test finished persisting bucket cache to file, renaming 2024-12-11T10:11:46,218 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T10:11:46,219 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T10:11:46,228 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestBucketCachePersister#testPrefetchListUponBlockEviction Thread=22 (was 21) Potentially hanging thread: hfile-prefetch-1733911904184 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=290 (was 289) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1374 (was 1442), ProcessCount=11 (was 11), AvailableMemoryMB=4381 (was 4458) 2024-12-11T10:11:46,234 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestBucketCachePersister#testPrefetchPersistenceCrash Thread=22, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=1374, ProcessCount=11, AvailableMemoryMB=4380 2024-12-11T10:11:46,235 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T10:11:46,239 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/testPrefetchPersistenceCrash 2024-12-11T10:11:46,239 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-11T10:11:46,240 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-11T10:11:46,240 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 3000 2024-12-11T10:11:46,240 INFO [Thread-25 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-11T10:11:46,240 WARN [Thread-25 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-11T10:11:46,241 INFO [Thread-25 {}] bucket.BucketAllocator(343): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-11T10:11:46,241 INFO [Thread-25 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/bucket.persistence is complete. 2024-12-11T10:11:46,259 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/testPrefetchPersistenceCrash, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-11T10:11:46,260 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T10:11:46,277 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 46412d32e5bb4cc790e500b614feca24 2024-12-11T10:11:46,287 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch0/46412d32e5bb4cc790e500b614feca24 is 223, key is aaaaaaaaaaaaaaaaaaaaaaabbaaabbaa/F4IyjRNVj:0giPN/-2403840707641718145/Put/seqid=0 2024-12-11T10:11:46,297 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b5c7940c1a15491f9886dce10bc6b6e7 2024-12-11T10:11:46,308 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch1/b5c7940c1a15491f9886dce10bc6b6e7 is 216, key is aaaaaaaaaaaaaaaaaaaaaaabaaabaabb/7_qa2:T4dc/3022311136953463842/Put/seqid=0 2024-12-11T10:11:46,311 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch0/46412d32e5bb4cc790e500b614feca24, delay=942 ms 2024-12-11T10:11:47,254 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_0. 2024-12-11T10:11:47,255 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_2187. 2024-12-11T10:11:47,256 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_4273. 2024-12-11T10:11:47,256 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_6401. 2024-12-11T10:11:47,257 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_8553. 2024-12-11T10:11:47,257 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_10753. 2024-12-11T10:11:47,257 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_12887. 2024-12-11T10:11:47,258 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_15001. 2024-12-11T10:11:47,258 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_17149. 2024-12-11T10:11:47,258 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_19241. 2024-12-11T10:11:47,259 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_21362. 2024-12-11T10:11:47,259 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_23455. 2024-12-11T10:11:47,259 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_25619. 2024-12-11T10:11:47,260 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_27821. 2024-12-11T10:11:47,260 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_29972. 2024-12-11T10:11:47,260 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_32075. 2024-12-11T10:11:47,261 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_34194. 2024-12-11T10:11:47,265 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_36300. 2024-12-11T10:11:47,266 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_38444. 2024-12-11T10:11:47,266 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_40604. 2024-12-11T10:11:47,266 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_42746. 2024-12-11T10:11:47,267 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_44978. 2024-12-11T10:11:47,267 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_47078. 2024-12-11T10:11:47,269 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_49254. 2024-12-11T10:11:47,269 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_51367. 2024-12-11T10:11:47,269 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_53486. 2024-12-11T10:11:47,270 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_55629. 2024-12-11T10:11:47,270 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_57781. 2024-12-11T10:11:47,271 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_59887. 2024-12-11T10:11:47,271 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_62041. 2024-12-11T10:11:47,271 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_64160. 2024-12-11T10:11:47,271 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_66383. 2024-12-11T10:11:47,272 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_68539. 2024-12-11T10:11:47,272 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_70640. 2024-12-11T10:11:47,272 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_72729. 2024-12-11T10:11:47,273 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_74848. 2024-12-11T10:11:47,273 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_76945. 2024-12-11T10:11:47,273 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_79080. 2024-12-11T10:11:47,274 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_81214. 2024-12-11T10:11:47,274 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_83339. 2024-12-11T10:11:47,274 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_85466. 2024-12-11T10:11:47,274 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_87624. 2024-12-11T10:11:47,275 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_89743. 2024-12-11T10:11:47,275 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_91962. 2024-12-11T10:11:47,275 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_94087. 2024-12-11T10:11:47,276 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_96174. 2024-12-11T10:11:47,276 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_98329. 2024-12-11T10:11:47,276 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_100443. 2024-12-11T10:11:47,276 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_102616. 2024-12-11T10:11:47,277 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_104765. 2024-12-11T10:11:47,277 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_106906. 2024-12-11T10:11:47,277 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_109115. 2024-12-11T10:11:47,278 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_111220. 2024-12-11T10:11:47,278 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_113346. 2024-12-11T10:11:47,278 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_115497. 2024-12-11T10:11:47,278 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_117635. 2024-12-11T10:11:47,279 DEBUG [hfile-prefetch-1733911900605 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46412d32e5bb4cc790e500b614feca24_117799. 2024-12-11T10:11:47,279 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch0/46412d32e5bb4cc790e500b614feca24, with total blocks 57, and data blocks 56 2024-12-11T10:11:47,279 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-11T10:11:47,279 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch0/46412d32e5bb4cc790e500b614feca24, but only 0 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue. 2024-12-11T10:11:47,279 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch0/46412d32e5bb4cc790e500b614feca24. Will sleep 100ms and try the verification again. 2024-12-11T10:11:47,380 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch0/46412d32e5bb4cc790e500b614feca24, with total blocks 57, and data blocks 56 2024-12-11T10:11:47,380 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2285): iterating over 17 entries in the backing map 2024-12-11T10:11:47,380 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 0 2024-12-11T10:11:47,380 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 2187 2024-12-11T10:11:47,380 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 4273 2024-12-11T10:11:47,380 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 6401 2024-12-11T10:11:47,381 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 8553 2024-12-11T10:11:47,381 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 34194 2024-12-11T10:11:47,381 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 36300 2024-12-11T10:11:47,381 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 49254 2024-12-11T10:11:47,381 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 55629 2024-12-11T10:11:47,381 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 72729 2024-12-11T10:11:47,381 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 79080 2024-12-11T10:11:47,381 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 87624 2024-12-11T10:11:47,382 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 91962 2024-12-11T10:11:47,382 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 96174 2024-12-11T10:11:47,382 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 100443 2024-12-11T10:11:47,382 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 113346 2024-12-11T10:11:47,382 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 115497 2024-12-11T10:11:47,383 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch0/46412d32e5bb4cc790e500b614feca24, but only 17 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue. 2024-12-11T10:11:47,383 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch0/46412d32e5bb4cc790e500b614feca24. Will sleep 100ms and try the verification again. 2024-12-11T10:11:47,483 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch0/46412d32e5bb4cc790e500b614feca24, with total blocks 57, and data blocks 56 2024-12-11T10:11:47,483 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2285): iterating over 57 entries in the backing map 2024-12-11T10:11:47,483 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 0 2024-12-11T10:11:47,483 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 2187 2024-12-11T10:11:47,483 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 4273 2024-12-11T10:11:47,483 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 6401 2024-12-11T10:11:47,484 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 8553 2024-12-11T10:11:47,484 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 10753 2024-12-11T10:11:47,484 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 12887 2024-12-11T10:11:47,484 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 15001 2024-12-11T10:11:47,484 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 17149 2024-12-11T10:11:47,484 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 19241 2024-12-11T10:11:47,484 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 21362 2024-12-11T10:11:47,484 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 23455 2024-12-11T10:11:47,484 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 25619 2024-12-11T10:11:47,484 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 27821 2024-12-11T10:11:47,484 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 29972 2024-12-11T10:11:47,485 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 32075 2024-12-11T10:11:47,485 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 34194 2024-12-11T10:11:47,485 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 36300 2024-12-11T10:11:47,485 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 38444 2024-12-11T10:11:47,485 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 40604 2024-12-11T10:11:47,485 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 42746 2024-12-11T10:11:47,485 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 44978 2024-12-11T10:11:47,485 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 47078 2024-12-11T10:11:47,485 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 49254 2024-12-11T10:11:47,485 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 51367 2024-12-11T10:11:47,485 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 53486 2024-12-11T10:11:47,485 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 55629 2024-12-11T10:11:47,485 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 57781 2024-12-11T10:11:47,486 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 59887 2024-12-11T10:11:47,486 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 62041 2024-12-11T10:11:47,486 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 64160 2024-12-11T10:11:47,486 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 66383 2024-12-11T10:11:47,486 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 68539 2024-12-11T10:11:47,486 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 70640 2024-12-11T10:11:47,486 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 72729 2024-12-11T10:11:47,486 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 74848 2024-12-11T10:11:47,486 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 76945 2024-12-11T10:11:47,486 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 79080 2024-12-11T10:11:47,487 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 81214 2024-12-11T10:11:47,487 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 83339 2024-12-11T10:11:47,487 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 85466 2024-12-11T10:11:47,487 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 87624 2024-12-11T10:11:47,487 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 89743 2024-12-11T10:11:47,487 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 91962 2024-12-11T10:11:47,487 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 94087 2024-12-11T10:11:47,487 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 96174 2024-12-11T10:11:47,487 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 98329 2024-12-11T10:11:47,487 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 100443 2024-12-11T10:11:47,487 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 102616 2024-12-11T10:11:47,487 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 104765 2024-12-11T10:11:47,488 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 106906 2024-12-11T10:11:47,488 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 109115 2024-12-11T10:11:47,488 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 111220 2024-12-11T10:11:47,488 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 113346 2024-12-11T10:11:47,488 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 115497 2024-12-11T10:11:47,488 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 117635 2024-12-11T10:11:47,488 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2293): found block for file 46412d32e5bb4cc790e500b614feca24 in the backing map. Acquiring read lock for offset 117799 2024-12-11T10:11:47,488 DEBUG [hfile-prefetch-1733911900605 {}] bucket.BucketCache(2305): File /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch0/46412d32e5bb4cc790e500b614feca24 has now been fully cached. 2024-12-11T10:11:47,489 DEBUG [hfile-prefetch-1733911900605 {}] hfile.PrefetchExecutor(121): Prefetch completed for 46412d32e5bb4cc790e500b614feca24 2024-12-11T10:11:48,314 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch1/b5c7940c1a15491f9886dce10bc6b6e7, delay=947 ms 2024-12-11T10:11:49,240 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-11T10:11:49,241 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-11T10:11:49,241 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-11T10:11:49,241 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 57, fullycachedFiles size: 1, chunkSize: 10000 2024-12-11T10:11:49,245 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 57, fullycachedFiles size: 1 2024-12-11T10:11:49,245 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 57, fullycachedFiles size: 1, file name: bucket.persistence1733911909241 2024-12-11T10:11:49,245 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-11T10:11:49,263 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_0. 2024-12-11T10:11:49,263 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_2171. 2024-12-11T10:11:49,264 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_4298. 2024-12-11T10:11:49,264 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_6395. 2024-12-11T10:11:49,264 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_8512. 2024-12-11T10:11:49,265 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_10703. 2024-12-11T10:11:49,265 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_12878. 2024-12-11T10:11:49,265 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_15017. 2024-12-11T10:11:49,265 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_17193. 2024-12-11T10:11:49,265 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_19407. 2024-12-11T10:11:49,266 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_21597. 2024-12-11T10:11:49,266 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_23773. 2024-12-11T10:11:49,266 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_25911. 2024-12-11T10:11:49,266 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_28010. 2024-12-11T10:11:49,267 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_30169. 2024-12-11T10:11:49,267 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_32288. 2024-12-11T10:11:49,267 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_34451. 2024-12-11T10:11:49,268 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_36621. 2024-12-11T10:11:49,268 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_38732. 2024-12-11T10:11:49,268 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_40856. 2024-12-11T10:11:49,268 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_42944. 2024-12-11T10:11:49,268 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_45158. 2024-12-11T10:11:49,269 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_47270. 2024-12-11T10:11:49,269 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_49363. 2024-12-11T10:11:49,269 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_51455. 2024-12-11T10:11:49,269 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_53625. 2024-12-11T10:11:49,270 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_55753. 2024-12-11T10:11:49,270 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_57864. 2024-12-11T10:11:49,270 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_59949. 2024-12-11T10:11:49,271 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_62085. 2024-12-11T10:11:49,271 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_64194. 2024-12-11T10:11:49,271 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_66326. 2024-12-11T10:11:49,272 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_68454. 2024-12-11T10:11:49,272 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_70583. 2024-12-11T10:11:49,274 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_72681. 2024-12-11T10:11:49,274 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_74865. 2024-12-11T10:11:49,274 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_76998. 2024-12-11T10:11:49,274 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_79154. 2024-12-11T10:11:49,275 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_81277. 2024-12-11T10:11:49,275 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_83424. 2024-12-11T10:11:49,275 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_85517. 2024-12-11T10:11:49,275 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_87616. 2024-12-11T10:11:49,275 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_89776. 2024-12-11T10:11:49,275 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_91927. 2024-12-11T10:11:49,276 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_94061. 2024-12-11T10:11:49,276 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_96173. 2024-12-11T10:11:49,276 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_98284. 2024-12-11T10:11:49,276 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_100502. 2024-12-11T10:11:49,276 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_102669. 2024-12-11T10:11:49,277 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_104848. 2024-12-11T10:11:49,277 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_107031. 2024-12-11T10:11:49,277 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_109183. 2024-12-11T10:11:49,277 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_111362. 2024-12-11T10:11:49,277 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_113448. 2024-12-11T10:11:49,277 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_115617. 2024-12-11T10:11:49,278 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_117785. 2024-12-11T10:11:49,278 DEBUG [hfile-prefetch-1733911904184 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b5c7940c1a15491f9886dce10bc6b6e7_118939. 2024-12-11T10:11:49,278 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch1/b5c7940c1a15491f9886dce10bc6b6e7, with total blocks 57, and data blocks 56 2024-12-11T10:11:49,278 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2285): iterating over 57 entries in the backing map 2024-12-11T10:11:49,278 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch1/b5c7940c1a15491f9886dce10bc6b6e7, but only 0 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue. 2024-12-11T10:11:49,278 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch1/b5c7940c1a15491f9886dce10bc6b6e7. Will sleep 100ms and try the verification again. 2024-12-11T10:11:49,378 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch1/b5c7940c1a15491f9886dce10bc6b6e7, with total blocks 57, and data blocks 56 2024-12-11T10:11:49,379 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2285): iterating over 57 entries in the backing map 2024-12-11T10:11:49,379 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch1/b5c7940c1a15491f9886dce10bc6b6e7, but only 0 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue. 2024-12-11T10:11:49,379 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch1/b5c7940c1a15491f9886dce10bc6b6e7. Will sleep 100ms and try the verification again. 2024-12-11T10:11:49,479 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch1/b5c7940c1a15491f9886dce10bc6b6e7, with total blocks 57, and data blocks 56 2024-12-11T10:11:49,479 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2285): iterating over 114 entries in the backing map 2024-12-11T10:11:49,479 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 0 2024-12-11T10:11:49,479 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 2171 2024-12-11T10:11:49,480 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 4298 2024-12-11T10:11:49,480 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 6395 2024-12-11T10:11:49,480 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 8512 2024-12-11T10:11:49,480 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 10703 2024-12-11T10:11:49,480 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 12878 2024-12-11T10:11:49,480 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 15017 2024-12-11T10:11:49,480 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 17193 2024-12-11T10:11:49,480 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 19407 2024-12-11T10:11:49,480 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 21597 2024-12-11T10:11:49,480 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 23773 2024-12-11T10:11:49,480 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 25911 2024-12-11T10:11:49,480 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 28010 2024-12-11T10:11:49,481 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 30169 2024-12-11T10:11:49,481 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 32288 2024-12-11T10:11:49,481 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 34451 2024-12-11T10:11:49,481 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 36621 2024-12-11T10:11:49,481 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 38732 2024-12-11T10:11:49,481 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 40856 2024-12-11T10:11:49,481 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 42944 2024-12-11T10:11:49,481 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 45158 2024-12-11T10:11:49,481 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 47270 2024-12-11T10:11:49,481 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 49363 2024-12-11T10:11:49,481 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 51455 2024-12-11T10:11:49,481 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 53625 2024-12-11T10:11:49,481 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 55753 2024-12-11T10:11:49,482 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 57864 2024-12-11T10:11:49,482 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 59949 2024-12-11T10:11:49,482 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 62085 2024-12-11T10:11:49,482 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 64194 2024-12-11T10:11:49,482 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 66326 2024-12-11T10:11:49,482 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 68454 2024-12-11T10:11:49,482 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 70583 2024-12-11T10:11:49,482 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 72681 2024-12-11T10:11:49,482 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 74865 2024-12-11T10:11:49,482 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 76998 2024-12-11T10:11:49,482 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 79154 2024-12-11T10:11:49,482 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 81277 2024-12-11T10:11:49,482 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 83424 2024-12-11T10:11:49,482 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 85517 2024-12-11T10:11:49,483 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 87616 2024-12-11T10:11:49,483 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 89776 2024-12-11T10:11:49,483 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 91927 2024-12-11T10:11:49,484 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 94061 2024-12-11T10:11:49,484 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 96173 2024-12-11T10:11:49,485 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 98284 2024-12-11T10:11:49,485 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 100502 2024-12-11T10:11:49,485 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 102669 2024-12-11T10:11:49,485 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 104848 2024-12-11T10:11:49,485 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 107031 2024-12-11T10:11:49,485 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 109183 2024-12-11T10:11:49,485 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 111362 2024-12-11T10:11:49,485 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 113448 2024-12-11T10:11:49,485 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 115617 2024-12-11T10:11:49,485 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 117785 2024-12-11T10:11:49,485 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2293): found block for file b5c7940c1a15491f9886dce10bc6b6e7 in the backing map. Acquiring read lock for offset 118939 2024-12-11T10:11:49,485 DEBUG [hfile-prefetch-1733911904184 {}] bucket.BucketCache(2305): File /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/TestPrefetch1/b5c7940c1a15491f9886dce10bc6b6e7 has now been fully cached. 2024-12-11T10:11:49,486 DEBUG [hfile-prefetch-1733911904184 {}] hfile.PrefetchExecutor(121): Prefetch completed for b5c7940c1a15491f9886dce10bc6b6e7 2024-12-11T10:11:52,245 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-11T10:11:52,246 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-11T10:11:52,246 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-11T10:11:52,246 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 114, fullycachedFiles size: 2, chunkSize: 10000 2024-12-11T10:11:52,250 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 114, fullycachedFiles size: 2 2024-12-11T10:11:52,250 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 114, fullycachedFiles size: 2, file name: bucket.persistence1733911912246 2024-12-11T10:11:52,250 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-11T10:11:53,317 INFO [Time-limited test {}] bucket.BucketCache(1738): Disabling cache 2024-12-11T10:11:53,318 INFO [Time-limited test {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6e6df6e1-408a-5bad-4c56-dd8c52038dc1/bucket.persistence 2024-12-11T10:11:53,318 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-11T10:11:53,318 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-11T10:11:53,318 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-11T10:11:53,318 INFO [Time-limited test {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-11T10:11:53,318 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-11T10:11:53,328 DEBUG [Time-limited test {}] bucket.BucketCache(1487): Thread Time-limited test started persisting bucket cache to file 2024-12-11T10:11:53,329 DEBUG [Time-limited test {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-11T10:11:53,329 DEBUG [Time-limited test {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 114, fullycachedFiles size: 2, chunkSize: 10000 2024-12-11T10:11:53,337 DEBUG [Time-limited test {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 114, fullycachedFiles size: 2 2024-12-11T10:11:53,338 DEBUG [Time-limited test {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 114, fullycachedFiles size: 2, file name: bucket.persistence1733911913328 2024-12-11T10:11:53,338 DEBUG [Time-limited test {}] bucket.BucketCache(1509): Thread Time-limited test finished persisting bucket cache to file, renaming 2024-12-11T10:11:53,338 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T10:11:53,339 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T10:11:53,346 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestBucketCachePersister#testPrefetchPersistenceCrash Thread=23 (was 22) Potentially hanging thread: hfile-prefetch-1733911906311 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=292 (was 290) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1320 (was 1374), ProcessCount=11 (was 11), AvailableMemoryMB=4232 (was 4380)