2024-12-10 10:50:37,327 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@57bc27f5 2024-12-10 10:50:37,346 main DEBUG Took 0.017918 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-10 10:50:37,347 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-10 10:50:37,347 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-10 10:50:37,348 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-10 10:50:37,349 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 10:50:37,357 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-10 10:50:37,371 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 10:50:37,372 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 10:50:37,373 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 10:50:37,374 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 10:50:37,374 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 10:50:37,375 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 10:50:37,376 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 10:50:37,376 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 10:50:37,377 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 10:50:37,377 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 10:50:37,378 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 10:50:37,379 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 10:50:37,380 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 10:50:37,380 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 10:50:37,381 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 10:50:37,381 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 10:50:37,382 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 10:50:37,382 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 10:50:37,382 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 10:50:37,383 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 10:50:37,383 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 10:50:37,383 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 10:50:37,384 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 10:50:37,384 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 10:50:37,384 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 10:50:37,385 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-10 10:50:37,386 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 10:50:37,387 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-10 10:50:37,389 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-10 10:50:37,389 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-10 10:50:37,390 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-10 10:50:37,390 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-10 10:50:37,399 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-10 10:50:37,401 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-10 10:50:37,402 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-10 10:50:37,402 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-10 10:50:37,403 main DEBUG createAppenders(={Console}) 2024-12-10 10:50:37,403 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@57bc27f5 initialized 2024-12-10 10:50:37,404 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@57bc27f5 2024-12-10 10:50:37,404 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@57bc27f5 OK. 2024-12-10 10:50:37,404 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-10 10:50:37,405 main DEBUG OutputStream closed 2024-12-10 10:50:37,405 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-10 10:50:37,405 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-10 10:50:37,405 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5a56cdac OK 2024-12-10 10:50:37,470 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-10 10:50:37,471 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-10 10:50:37,472 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-10 10:50:37,473 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-10 10:50:37,474 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-10 10:50:37,474 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-10 10:50:37,474 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-10 10:50:37,475 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-10 10:50:37,475 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-10 10:50:37,475 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-10 10:50:37,476 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-10 10:50:37,476 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-10 10:50:37,476 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-10 10:50:37,477 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-10 10:50:37,477 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-10 10:50:37,477 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-10 10:50:37,478 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-10 10:50:37,478 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-10 10:50:37,480 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-10 10:50:37,481 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@1608bcbd) with optional ClassLoader: null 2024-12-10 10:50:37,481 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-10 10:50:37,482 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@1608bcbd] started OK. 2024-12-10T10:50:37,495 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.bucket.TestBucketCachePersister timeout: 13 mins 2024-12-10 10:50:37,498 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-10 10:50:37,498 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-10T10:50:37,732 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088 2024-12-10T10:50:37,755 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestBucketCachePersister#testPrefetchBlockEvictionWhilePrefetchRunning Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=170, ProcessCount=11, AvailableMemoryMB=4624 2024-12-10T10:50:37,847 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-10T10:50:37,932 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T10:50:37,963 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/testPrefetchBlockEvictionWhilePrefetchRunning 2024-12-10T10:50:37,972 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-10T10:50:37,975 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-10T10:50:37,977 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 200 2024-12-10T10:50:37,979 INFO [Thread-5 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-10T10:50:37,979 WARN [Thread-5 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-10T10:50:37,985 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/testPrefetchBlockEvictionWhilePrefetchRunning, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-10T10:50:37,987 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T10:50:37,990 INFO [Thread-5 {}] bucket.BucketAllocator(343): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-10T10:50:37,991 INFO [Thread-5 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/bucket.persistence is complete. 2024-12-10T10:50:38,012 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-10T10:50:38,071 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a8941c2476294177838d23e1456c8667 2024-12-10T10:50:38,165 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T10:50:38,201 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-10T10:50:38,234 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is 227, key is aaaaaaaaaaaaaaaaaaaaaaaaaaabbaaa/h0:NhOEzQ7/-8308077059869649705/Put/seqid=0 2024-12-10T10:50:38,339 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667, delay=981 ms 2024-12-10T10:50:38,377 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,387 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,397 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,408 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,418 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,428 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,439 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,449 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,459 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,470 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,480 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,490 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,500 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,511 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,521 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,531 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,541 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,552 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,562 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,572 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,583 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,593 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,603 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,613 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,624 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,634 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,644 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,654 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,665 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,675 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,685 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,696 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,706 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,716 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,726 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,737 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,747 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,757 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,767 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,778 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,788 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,798 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,808 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,819 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,829 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,839 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,850 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,860 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,870 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,880 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,891 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,901 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,911 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,922 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,932 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,942 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,952 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,963 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,973 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,983 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:38,994 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:39,004 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:39,014 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:39,024 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:39,035 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:39,045 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:39,055 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:39,066 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:39,076 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:39,086 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:39,096 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:39,107 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:39,117 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:39,127 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:39,138 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:39,148 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:39,158 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:39,168 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:39,178 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:39,189 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:39,200 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:39,210 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:39,220 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:39,230 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:39,241 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:39,251 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:39,261 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:39,271 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:39,282 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:39,292 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:39,302 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:39,312 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:39,323 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:39,333 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:39,332 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_0. 2024-12-10T10:50:39,343 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:39,354 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:39,364 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:39,366 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_2165. 2024-12-10T10:50:39,366 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_4323. 2024-12-10T10:50:39,367 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_6453. 2024-12-10T10:50:39,367 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_8614. 2024-12-10T10:50:39,367 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_10807. 2024-12-10T10:50:39,368 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_12947. 2024-12-10T10:50:39,368 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_15075. 2024-12-10T10:50:39,369 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_17290. 2024-12-10T10:50:39,369 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_19464. 2024-12-10T10:50:39,370 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_21630. 2024-12-10T10:50:39,370 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_23834. 2024-12-10T10:50:39,370 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_26039. 2024-12-10T10:50:39,371 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_28143. 2024-12-10T10:50:39,371 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_30241. 2024-12-10T10:50:39,371 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_32352. 2024-12-10T10:50:39,372 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_34485. 2024-12-10T10:50:39,372 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_36643. 2024-12-10T10:50:39,373 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_38854. 2024-12-10T10:50:39,373 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_40954. 2024-12-10T10:50:39,374 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_43160. 2024-12-10T10:50:39,374 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_45339. 2024-12-10T10:50:39,374 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:39,375 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_47445. 2024-12-10T10:50:39,375 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_49558. 2024-12-10T10:50:39,375 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_51722. 2024-12-10T10:50:39,376 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_53936. 2024-12-10T10:50:39,376 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_56024. 2024-12-10T10:50:39,376 DEBUG [Time-limited test {}] bucket.BucketCache(856): removed key a8941c2476294177838d23e1456c8667_0 from back map with offset lock 522240 in the evict process 2024-12-10T10:50:39,377 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_58140. 2024-12-10T10:50:39,377 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(182): Attempted eviction for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667. Succeeded? true 2024-12-10T10:50:39,377 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_60236. 2024-12-10T10:50:39,378 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_62470. 2024-12-10T10:50:39,378 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_64587. 2024-12-10T10:50:39,378 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-10T10:50:39,378 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-10T10:50:39,379 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_66698. 2024-12-10T10:50:39,379 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-10T10:50:39,379 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 20, fullycachedFiles size: 0, chunkSize: 10000 2024-12-10T10:50:39,379 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_68833. 2024-12-10T10:50:39,379 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_70932. 2024-12-10T10:50:39,380 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_73123. 2024-12-10T10:50:39,380 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_75357. 2024-12-10T10:50:39,381 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_77518. 2024-12-10T10:50:39,381 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_79652. 2024-12-10T10:50:39,382 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_81771. 2024-12-10T10:50:39,382 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_83936. 2024-12-10T10:50:39,383 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_86157. 2024-12-10T10:50:39,383 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_88257. 2024-12-10T10:50:39,384 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_90363. 2024-12-10T10:50:39,384 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_92539. 2024-12-10T10:50:39,385 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_94694. 2024-12-10T10:50:39,385 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_96859. 2024-12-10T10:50:39,385 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_98960. 2024-12-10T10:50:39,386 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_101072. 2024-12-10T10:50:39,386 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_103251. 2024-12-10T10:50:39,387 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_105366. 2024-12-10T10:50:39,387 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:39,387 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_107540. 2024-12-10T10:50:39,387 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_109658. 2024-12-10T10:50:39,388 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_111783. 2024-12-10T10:50:39,388 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_113886. 2024-12-10T10:50:39,389 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_115984. 2024-12-10T10:50:39,389 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_118082. 2024-12-10T10:50:39,390 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a8941c2476294177838d23e1456c8667_118341. 2024-12-10T10:50:39,390 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667, with total blocks 57, and data blocks 56 2024-12-10T10:50:39,390 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2285): iterating over 46 entries in the backing map 2024-12-10T10:50:39,391 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 2165 2024-12-10T10:50:39,392 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 4323 2024-12-10T10:50:39,392 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 6453 2024-12-10T10:50:39,392 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 8614 2024-12-10T10:50:39,392 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 10807 2024-12-10T10:50:39,392 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 12947 2024-12-10T10:50:39,393 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 15075 2024-12-10T10:50:39,393 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 17290 2024-12-10T10:50:39,393 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 19464 2024-12-10T10:50:39,393 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 21630 2024-12-10T10:50:39,393 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 23834 2024-12-10T10:50:39,393 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 26039 2024-12-10T10:50:39,393 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 28143 2024-12-10T10:50:39,394 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 30241 2024-12-10T10:50:39,394 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 32352 2024-12-10T10:50:39,394 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 34485 2024-12-10T10:50:39,394 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 36643 2024-12-10T10:50:39,394 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 38854 2024-12-10T10:50:39,394 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 40954 2024-12-10T10:50:39,394 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 43160 2024-12-10T10:50:39,394 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 45339 2024-12-10T10:50:39,395 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 47445 2024-12-10T10:50:39,395 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 49558 2024-12-10T10:50:39,395 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 51722 2024-12-10T10:50:39,395 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 53936 2024-12-10T10:50:39,395 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 56024 2024-12-10T10:50:39,395 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 58140 2024-12-10T10:50:39,395 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 60236 2024-12-10T10:50:39,396 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 62470 2024-12-10T10:50:39,396 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 64587 2024-12-10T10:50:39,396 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 66698 2024-12-10T10:50:39,396 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 68833 2024-12-10T10:50:39,396 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 70932 2024-12-10T10:50:39,396 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 73123 2024-12-10T10:50:39,396 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 75357 2024-12-10T10:50:39,396 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 77518 2024-12-10T10:50:39,396 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 79652 2024-12-10T10:50:39,397 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 81771 2024-12-10T10:50:39,397 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 83936 2024-12-10T10:50:39,397 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 86157 2024-12-10T10:50:39,397 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 88257 2024-12-10T10:50:39,397 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 90363 2024-12-10T10:50:39,397 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 92539 2024-12-10T10:50:39,397 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667 is still running. 2024-12-10T10:50:39,397 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 94694 2024-12-10T10:50:39,398 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 96859 2024-12-10T10:50:39,398 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 98960 2024-12-10T10:50:39,398 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 101072 2024-12-10T10:50:39,398 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 103251 2024-12-10T10:50:39,398 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 105366 2024-12-10T10:50:39,398 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 107540 2024-12-10T10:50:39,398 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 109658 2024-12-10T10:50:39,398 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 111783 2024-12-10T10:50:39,398 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 113886 2024-12-10T10:50:39,399 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 115984 2024-12-10T10:50:39,399 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 118082 2024-12-10T10:50:39,399 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file a8941c2476294177838d23e1456c8667 in the backing map. Acquiring read lock for offset 118341 2024-12-10T10:50:39,399 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667, but only 55 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue. 2024-12-10T10:50:39,399 INFO [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2331): We found only 55 data blocks cached from a total of 56 for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/a8941c2476294177838d23e1456c8667, but no blocks pending caching. Maybe cache is full or evictions happened concurrently to cache prefetch. 2024-12-10T10:50:39,400 DEBUG [hfile-prefetch-1733827838340 {}] hfile.PrefetchExecutor(121): Prefetch completed for a8941c2476294177838d23e1456c8667 2024-12-10T10:50:39,408 INFO [Time-limited test {}] bucket.BucketCache(1738): Disabling cache 2024-12-10T10:50:39,408 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-10T10:50:39,408 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-10T10:50:39,408 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-10T10:50:39,410 INFO [Time-limited test {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/bucket.persistence 2024-12-10T10:50:39,410 INFO [Time-limited test {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-10T10:50:39,468 ERROR [bucket-cache-persister {}] bucket.PersistentIOEngine(78): Calculating checksum failed, because of java.io.InterruptedIOException: java.lang.InterruptedException at org.apache.hadoop.util.Shell.runCommand(Shell.java:1071) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.getFileSize(PersistentIOEngine.java:94) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.calculateChecksum(PersistentIOEngine.java:71) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.toPB(BucketProtoUtils.java:61) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.serializeAsPB(BucketProtoUtils.java:75) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistChunkedBackingMap(BucketCache.java:1678) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1496) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] Caused by: java.lang.InterruptedException at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at java.lang.ProcessImpl.waitFor(ProcessImpl.java:434) ~[?:?] at org.apache.hadoop.util.Shell.runCommand(Shell.java:1061) ~[hadoop-common-3.4.1.jar:?] ... 9 more 2024-12-10T10:50:39,480 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 56, fullycachedFiles size: 0 2024-12-10T10:50:39,480 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 56, fullycachedFiles size: 0, file name: bucket.persistence1733827839378 2024-12-10T10:50:39,481 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-10T10:50:39,481 INFO [bucket-cache-persister {}] bucket.BucketCachePersister(62): Finishing cache persister thread. 2024-12-10T10:50:39,490 DEBUG [Time-limited test {}] bucket.BucketCache(1487): Thread Time-limited test started persisting bucket cache to file 2024-12-10T10:50:39,491 DEBUG [Time-limited test {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-10T10:50:39,491 DEBUG [Time-limited test {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 56, fullycachedFiles size: 0, chunkSize: 10000 2024-12-10T10:50:39,496 DEBUG [Time-limited test {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 56, fullycachedFiles size: 0 2024-12-10T10:50:39,496 DEBUG [Time-limited test {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 56, fullycachedFiles size: 0, file name: bucket.persistence1733827839491 2024-12-10T10:50:39,496 DEBUG [Time-limited test {}] bucket.BucketCache(1509): Thread Time-limited test finished persisting bucket cache to file, renaming 2024-12-10T10:50:39,497 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T10:50:39,497 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T10:50:39,501 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T10:50:39,507 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestBucketCachePersister#testPrefetchBlockEvictionWhilePrefetchRunning Thread=20 (was 12) Potentially hanging thread: hfile-prefetch-1733827838340 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) - Thread LEAK? -, OpenFileDescriptor=288 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=170 (was 170), ProcessCount=11 (was 11), AvailableMemoryMB=4579 (was 4624) 2024-12-10T10:50:39,513 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestBucketCachePersister#testPrefetchPersistenceCrashNegative Thread=20, OpenFileDescriptor=288, MaxFileDescriptor=1048576, SystemLoadAverage=170, ProcessCount=11, AvailableMemoryMB=4579 2024-12-10T10:50:39,513 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T10:50:39,517 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/testPrefetchPersistenceCrashNegative 2024-12-10T10:50:39,517 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-10T10:50:39,517 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-10T10:50:39,517 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 9223372036854775807 2024-12-10T10:50:39,518 INFO [Thread-13 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-10T10:50:39,518 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/testPrefetchPersistenceCrashNegative, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-10T10:50:39,518 WARN [Thread-13 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-10T10:50:39,518 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T10:50:39,518 INFO [Thread-13 {}] bucket.BucketAllocator(343): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-10T10:50:39,519 INFO [Thread-13 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/bucket.persistence is complete. 2024-12-10T10:50:39,525 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 591fd492c26f4b4491264e4990d62f9d 2024-12-10T10:50:39,544 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch2/591fd492c26f4b4491264e4990d62f9d is 230, key is aaaaaaaaaaaaaaaaaaaaaabaababbaab/YmeLAel:XxUf/-5715261506732487393/Put/seqid=0 2024-12-10T10:50:39,547 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch2/591fd492c26f4b4491264e4990d62f9d, delay=996 ms 2024-12-10T10:50:40,544 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_0. 2024-12-10T10:50:40,545 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_2132. 2024-12-10T10:50:40,545 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_4368. 2024-12-10T10:50:40,546 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_6465. 2024-12-10T10:50:40,546 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_8621. 2024-12-10T10:50:40,547 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_10778. 2024-12-10T10:50:40,547 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_12922. 2024-12-10T10:50:40,547 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_15055. 2024-12-10T10:50:40,548 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_17163. 2024-12-10T10:50:40,548 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_19300. 2024-12-10T10:50:40,549 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_21420. 2024-12-10T10:50:40,549 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_23572. 2024-12-10T10:50:40,549 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_25736. 2024-12-10T10:50:40,550 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_27905. 2024-12-10T10:50:40,550 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_30015. 2024-12-10T10:50:40,551 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_32102. 2024-12-10T10:50:40,551 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_34195. 2024-12-10T10:50:40,551 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_36296. 2024-12-10T10:50:40,552 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_38466. 2024-12-10T10:50:40,552 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_40689. 2024-12-10T10:50:40,552 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_42887. 2024-12-10T10:50:40,553 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_45081. 2024-12-10T10:50:40,553 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_47265. 2024-12-10T10:50:40,553 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_49363. 2024-12-10T10:50:40,554 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_51470. 2024-12-10T10:50:40,554 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_53663. 2024-12-10T10:50:40,554 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_55843. 2024-12-10T10:50:40,555 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_57967. 2024-12-10T10:50:40,555 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_60165. 2024-12-10T10:50:40,555 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_62291. 2024-12-10T10:50:40,556 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_64425. 2024-12-10T10:50:40,556 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_66515. 2024-12-10T10:50:40,556 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_68765. 2024-12-10T10:50:40,557 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_70884. 2024-12-10T10:50:40,557 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_73040. 2024-12-10T10:50:40,557 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_75185. 2024-12-10T10:50:40,558 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_77306. 2024-12-10T10:50:40,558 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_79401. 2024-12-10T10:50:40,558 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_81586. 2024-12-10T10:50:40,559 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_83706. 2024-12-10T10:50:40,559 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_85856. 2024-12-10T10:50:40,559 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_87948. 2024-12-10T10:50:40,560 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_90179. 2024-12-10T10:50:40,560 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_92294. 2024-12-10T10:50:40,560 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_94466. 2024-12-10T10:50:40,561 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_96592. 2024-12-10T10:50:40,561 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_98736. 2024-12-10T10:50:40,562 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_100850. 2024-12-10T10:50:40,562 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_103021. 2024-12-10T10:50:40,562 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_105131. 2024-12-10T10:50:40,563 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_107232. 2024-12-10T10:50:40,563 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_109385. 2024-12-10T10:50:40,563 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_111567. 2024-12-10T10:50:40,564 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_113780. 2024-12-10T10:50:40,564 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_115970. 2024-12-10T10:50:40,564 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 591fd492c26f4b4491264e4990d62f9d_116489. 2024-12-10T10:50:40,565 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch2/591fd492c26f4b4491264e4990d62f9d, with total blocks 56, and data blocks 55 2024-12-10T10:50:40,565 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2285): iterating over 41 entries in the backing map 2024-12-10T10:50:40,565 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 0 2024-12-10T10:50:40,565 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 2132 2024-12-10T10:50:40,565 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 4368 2024-12-10T10:50:40,565 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 6465 2024-12-10T10:50:40,565 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 8621 2024-12-10T10:50:40,566 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 10778 2024-12-10T10:50:40,566 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 12922 2024-12-10T10:50:40,566 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 15055 2024-12-10T10:50:40,566 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 17163 2024-12-10T10:50:40,566 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 19300 2024-12-10T10:50:40,566 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 21420 2024-12-10T10:50:40,566 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 23572 2024-12-10T10:50:40,566 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 25736 2024-12-10T10:50:40,566 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 27905 2024-12-10T10:50:40,566 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 30015 2024-12-10T10:50:40,567 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 32102 2024-12-10T10:50:40,567 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 34195 2024-12-10T10:50:40,567 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 36296 2024-12-10T10:50:40,567 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 38466 2024-12-10T10:50:40,567 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 40689 2024-12-10T10:50:40,567 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 42887 2024-12-10T10:50:40,567 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 45081 2024-12-10T10:50:40,567 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 47265 2024-12-10T10:50:40,567 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 49363 2024-12-10T10:50:40,567 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 51470 2024-12-10T10:50:40,567 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 53663 2024-12-10T10:50:40,568 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 55843 2024-12-10T10:50:40,568 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 57967 2024-12-10T10:50:40,568 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 60165 2024-12-10T10:50:40,568 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 62291 2024-12-10T10:50:40,568 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 64425 2024-12-10T10:50:40,568 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 66515 2024-12-10T10:50:40,568 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 68765 2024-12-10T10:50:40,568 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 70884 2024-12-10T10:50:40,569 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 73040 2024-12-10T10:50:40,569 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 75185 2024-12-10T10:50:40,569 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 77306 2024-12-10T10:50:40,569 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 79401 2024-12-10T10:50:40,569 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 81586 2024-12-10T10:50:40,569 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 83706 2024-12-10T10:50:40,569 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 85856 2024-12-10T10:50:40,569 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 87948 2024-12-10T10:50:40,569 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 90179 2024-12-10T10:50:40,569 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 92294 2024-12-10T10:50:40,569 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 94466 2024-12-10T10:50:40,570 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 96592 2024-12-10T10:50:40,570 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 98736 2024-12-10T10:50:40,570 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 100850 2024-12-10T10:50:40,570 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 103021 2024-12-10T10:50:40,570 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 105131 2024-12-10T10:50:40,570 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 107232 2024-12-10T10:50:40,570 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 109385 2024-12-10T10:50:40,570 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 111567 2024-12-10T10:50:40,570 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 113780 2024-12-10T10:50:40,570 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 115970 2024-12-10T10:50:40,570 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file 591fd492c26f4b4491264e4990d62f9d in the backing map. Acquiring read lock for offset 116489 2024-12-10T10:50:40,570 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2305): File /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch2/591fd492c26f4b4491264e4990d62f9d has now been fully cached. 2024-12-10T10:50:40,571 DEBUG [hfile-prefetch-1733827838340 {}] hfile.PrefetchExecutor(121): Prefetch completed for 591fd492c26f4b4491264e4990d62f9d 2024-12-10T10:50:41,551 INFO [Time-limited test {}] bucket.BucketCache(1738): Disabling cache 2024-12-10T10:50:41,551 INFO [Time-limited test {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/bucket.persistence 2024-12-10T10:50:41,551 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-10T10:50:41,551 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-10T10:50:41,551 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-10T10:50:41,552 INFO [Time-limited test {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-10T10:50:41,552 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-10T10:50:41,562 DEBUG [Time-limited test {}] bucket.BucketCache(1487): Thread Time-limited test started persisting bucket cache to file 2024-12-10T10:50:41,562 DEBUG [Time-limited test {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-10T10:50:41,562 DEBUG [Time-limited test {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 56, fullycachedFiles size: 1, chunkSize: 10000 2024-12-10T10:50:41,567 DEBUG [Time-limited test {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 56, fullycachedFiles size: 1 2024-12-10T10:50:41,567 DEBUG [Time-limited test {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 56, fullycachedFiles size: 1, file name: bucket.persistence1733827841562 2024-12-10T10:50:41,567 DEBUG [Time-limited test {}] bucket.BucketCache(1509): Thread Time-limited test finished persisting bucket cache to file, renaming 2024-12-10T10:50:41,568 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T10:50:41,568 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T10:50:41,575 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestBucketCachePersister#testPrefetchPersistenceCrashNegative Thread=21 (was 20) Potentially hanging thread: hfile-prefetch-1733827839547 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=289 (was 288) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=170 (was 170), ProcessCount=11 (was 11), AvailableMemoryMB=4576 (was 4579) 2024-12-10T10:50:41,580 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestBucketCachePersister#testPrefetchListUponBlockEviction Thread=21, OpenFileDescriptor=289, MaxFileDescriptor=1048576, SystemLoadAverage=170, ProcessCount=11, AvailableMemoryMB=4576 2024-12-10T10:50:41,581 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T10:50:41,584 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/testPrefetchListUponBlockEviction 2024-12-10T10:50:41,584 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-10T10:50:41,584 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-10T10:50:41,584 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 200 2024-12-10T10:50:41,585 INFO [Thread-18 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-10T10:50:41,585 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/testPrefetchListUponBlockEviction, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-10T10:50:41,585 WARN [Thread-18 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-10T10:50:41,585 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T10:50:41,585 INFO [Thread-18 {}] bucket.BucketAllocator(343): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-10T10:50:41,585 INFO [Thread-18 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/bucket.persistence is complete. 2024-12-10T10:50:41,595 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 0653587d02dd4da8be058096fb98fe78 2024-12-10T10:50:41,604 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/0653587d02dd4da8be058096fb98fe78 is 244, key is aaaaaaaaaaaaaaaaaaaaaaabbabbbaab/rpb:ZFMclaSfPAv/1845673983316745965/DeleteFamily/seqid=0 2024-12-10T10:50:41,606 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/0653587d02dd4da8be058096fb98fe78, delay=923 ms 2024-12-10T10:50:41,785 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-10T10:50:41,785 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-10T10:50:41,785 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-10T10:50:41,785 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-10T10:50:41,789 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-10T10:50:41,789 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733827841785 2024-12-10T10:50:41,789 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-10T10:50:42,531 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_0. 2024-12-10T10:50:42,531 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_2141. 2024-12-10T10:50:42,532 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_4314. 2024-12-10T10:50:42,532 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_6488. 2024-12-10T10:50:42,533 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_8613. 2024-12-10T10:50:42,533 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_10832. 2024-12-10T10:50:42,533 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_13054. 2024-12-10T10:50:42,533 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_15215. 2024-12-10T10:50:42,534 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_17388. 2024-12-10T10:50:42,534 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_19665. 2024-12-10T10:50:42,534 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_21816. 2024-12-10T10:50:42,535 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_23938. 2024-12-10T10:50:42,535 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_26118. 2024-12-10T10:50:42,535 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_28243. 2024-12-10T10:50:42,536 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_30336. 2024-12-10T10:50:42,536 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_32459. 2024-12-10T10:50:42,537 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_34559. 2024-12-10T10:50:42,537 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_36648. 2024-12-10T10:50:42,537 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_38908. 2024-12-10T10:50:42,538 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_41073. 2024-12-10T10:50:42,538 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_43206. 2024-12-10T10:50:42,539 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_45337. 2024-12-10T10:50:42,539 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_47477. 2024-12-10T10:50:42,539 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_49591. 2024-12-10T10:50:42,540 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_51692. 2024-12-10T10:50:42,540 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_53908. 2024-12-10T10:50:42,541 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_56043. 2024-12-10T10:50:42,541 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_58247. 2024-12-10T10:50:42,541 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_60414. 2024-12-10T10:50:42,542 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_62582. 2024-12-10T10:50:42,542 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_64720. 2024-12-10T10:50:42,543 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_66912. 2024-12-10T10:50:42,543 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_69128. 2024-12-10T10:50:42,543 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_71250. 2024-12-10T10:50:42,544 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_73349. 2024-12-10T10:50:42,544 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_75515. 2024-12-10T10:50:42,544 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_77670. 2024-12-10T10:50:42,545 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_79888. 2024-12-10T10:50:42,545 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_82030. 2024-12-10T10:50:42,546 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_84178. 2024-12-10T10:50:42,546 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_86364. 2024-12-10T10:50:42,547 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_88567. 2024-12-10T10:50:42,547 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_90686. 2024-12-10T10:50:42,548 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_92890. 2024-12-10T10:50:42,548 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_95054. 2024-12-10T10:50:42,548 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_97238. 2024-12-10T10:50:42,549 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_99358. 2024-12-10T10:50:42,549 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_101507. 2024-12-10T10:50:42,550 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_103635. 2024-12-10T10:50:42,550 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_105795. 2024-12-10T10:50:42,550 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_107903. 2024-12-10T10:50:42,551 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_110085. 2024-12-10T10:50:42,551 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_112178. 2024-12-10T10:50:42,552 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_114360. 2024-12-10T10:50:42,552 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_116487. 2024-12-10T10:50:42,552 DEBUG [hfile-prefetch-1733827839547 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0653587d02dd4da8be058096fb98fe78_118610. 2024-12-10T10:50:42,553 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/0653587d02dd4da8be058096fb98fe78, with total blocks 56, and data blocks 55 2024-12-10T10:50:42,553 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2285): iterating over 43 entries in the backing map 2024-12-10T10:50:42,553 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 0 2024-12-10T10:50:42,553 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 2141 2024-12-10T10:50:42,553 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 4314 2024-12-10T10:50:42,553 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 6488 2024-12-10T10:50:42,554 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 8613 2024-12-10T10:50:42,554 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 10832 2024-12-10T10:50:42,554 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 13054 2024-12-10T10:50:42,554 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 15215 2024-12-10T10:50:42,554 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 17388 2024-12-10T10:50:42,554 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 19665 2024-12-10T10:50:42,554 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 21816 2024-12-10T10:50:42,554 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 23938 2024-12-10T10:50:42,554 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 26118 2024-12-10T10:50:42,554 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 28243 2024-12-10T10:50:42,554 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 30336 2024-12-10T10:50:42,555 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 32459 2024-12-10T10:50:42,555 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 34559 2024-12-10T10:50:42,555 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 36648 2024-12-10T10:50:42,555 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 38908 2024-12-10T10:50:42,555 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 41073 2024-12-10T10:50:42,555 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 43206 2024-12-10T10:50:42,556 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 45337 2024-12-10T10:50:42,556 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 47477 2024-12-10T10:50:42,556 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 49591 2024-12-10T10:50:42,556 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 51692 2024-12-10T10:50:42,556 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 53908 2024-12-10T10:50:42,556 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 56043 2024-12-10T10:50:42,556 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 58247 2024-12-10T10:50:42,556 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 60414 2024-12-10T10:50:42,556 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 62582 2024-12-10T10:50:42,556 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 64720 2024-12-10T10:50:42,557 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 66912 2024-12-10T10:50:42,557 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 69128 2024-12-10T10:50:42,557 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 71250 2024-12-10T10:50:42,557 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 73349 2024-12-10T10:50:42,557 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 75515 2024-12-10T10:50:42,557 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 77670 2024-12-10T10:50:42,557 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 79888 2024-12-10T10:50:42,557 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 82030 2024-12-10T10:50:42,557 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 84178 2024-12-10T10:50:42,557 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 86364 2024-12-10T10:50:42,558 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 88567 2024-12-10T10:50:42,558 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 90686 2024-12-10T10:50:42,558 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 92890 2024-12-10T10:50:42,558 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 95054 2024-12-10T10:50:42,558 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 97238 2024-12-10T10:50:42,558 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 99358 2024-12-10T10:50:42,558 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 101507 2024-12-10T10:50:42,558 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 103635 2024-12-10T10:50:42,558 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 105795 2024-12-10T10:50:42,558 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 107903 2024-12-10T10:50:42,559 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 110085 2024-12-10T10:50:42,559 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 112178 2024-12-10T10:50:42,559 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 114360 2024-12-10T10:50:42,559 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 116487 2024-12-10T10:50:42,559 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2293): found block for file 0653587d02dd4da8be058096fb98fe78 in the backing map. Acquiring read lock for offset 118610 2024-12-10T10:50:42,559 DEBUG [hfile-prefetch-1733827839547 {}] bucket.BucketCache(2305): File /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch3/0653587d02dd4da8be058096fb98fe78 has now been fully cached. 2024-12-10T10:50:42,560 DEBUG [hfile-prefetch-1733827839547 {}] hfile.PrefetchExecutor(121): Prefetch completed for 0653587d02dd4da8be058096fb98fe78 2024-12-10T10:50:42,590 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-10T10:50:42,590 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-10T10:50:42,590 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-10T10:50:42,590 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 56, fullycachedFiles size: 1, chunkSize: 10000 2024-12-10T10:50:42,594 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 56, fullycachedFiles size: 1 2024-12-10T10:50:42,594 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 56, fullycachedFiles size: 1, file name: bucket.persistence1733827842590 2024-12-10T10:50:42,594 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-10T10:50:42,608 DEBUG [Time-limited test {}] bucket.BucketCache(856): removed key 0653587d02dd4da8be058096fb98fe78_38908 from back map with offset lock 470016 in the evict process 2024-12-10T10:50:42,608 DEBUG [Time-limited test {}] bucket.BucketCache(768): Removing file 0653587d02dd4da8be058096fb98fe78 for region 702be8da-b702-3ba1-3bea-f8df079f1088 2024-12-10T10:50:42,608 INFO [Time-limited test {}] bucket.BucketCache(1738): Disabling cache 2024-12-10T10:50:42,609 INFO [Time-limited test {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/bucket.persistence 2024-12-10T10:50:42,609 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-10T10:50:42,609 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-10T10:50:42,609 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-10T10:50:42,609 INFO [Time-limited test {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-10T10:50:42,609 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-10T10:50:42,619 DEBUG [Time-limited test {}] bucket.BucketCache(1487): Thread Time-limited test started persisting bucket cache to file 2024-12-10T10:50:42,619 DEBUG [Time-limited test {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-10T10:50:42,619 DEBUG [Time-limited test {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 55, fullycachedFiles size: 0, chunkSize: 10000 2024-12-10T10:50:42,623 DEBUG [Time-limited test {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 55, fullycachedFiles size: 0 2024-12-10T10:50:42,623 DEBUG [Time-limited test {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 55, fullycachedFiles size: 0, file name: bucket.persistence1733827842619 2024-12-10T10:50:42,623 DEBUG [Time-limited test {}] bucket.BucketCache(1509): Thread Time-limited test finished persisting bucket cache to file, renaming 2024-12-10T10:50:42,624 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T10:50:42,625 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T10:50:42,631 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestBucketCachePersister#testPrefetchListUponBlockEviction Thread=22 (was 21) Potentially hanging thread: hfile-prefetch-1733827841606 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=290 (was 289) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=156 (was 170), ProcessCount=11 (was 11), AvailableMemoryMB=4577 (was 4576) - AvailableMemoryMB LEAK? - 2024-12-10T10:50:42,636 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestBucketCachePersister#testPrefetchPersistenceCrash Thread=22, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=156, ProcessCount=11, AvailableMemoryMB=4577 2024-12-10T10:50:42,636 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T10:50:42,640 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/testPrefetchPersistenceCrash 2024-12-10T10:50:42,640 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-10T10:50:42,640 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-10T10:50:42,640 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 3000 2024-12-10T10:50:42,640 INFO [Thread-25 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-10T10:50:42,640 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/testPrefetchPersistenceCrash, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-10T10:50:42,640 WARN [Thread-25 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-10T10:50:42,640 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T10:50:42,641 INFO [Thread-25 {}] bucket.BucketAllocator(343): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-10T10:50:42,641 INFO [Thread-25 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/bucket.persistence is complete. 2024-12-10T10:50:42,647 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d15845c9e82d4e39be52b6842d45b1a5 2024-12-10T10:50:42,654 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch0/d15845c9e82d4e39be52b6842d45b1a5 is 228, key is aaaaaaaaaaaaaaaaaaaaaaaaaabbbabb/0XQaW2lMG:/634151812345075405/DeleteFamily/seqid=0 2024-12-10T10:50:42,661 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 299160daa1ed4f08b95094e277d652fc 2024-12-10T10:50:42,668 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch1/299160daa1ed4f08b95094e277d652fc is 238, key is aaaaaaaaaaaaaaaaaaaaaaaaabababbb/Q2J:9cr/-865784010673016708/DeleteColumn/seqid=0 2024-12-10T10:50:42,670 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch0/d15845c9e82d4e39be52b6842d45b1a5, delay=945 ms 2024-12-10T10:50:43,616 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_0. 2024-12-10T10:50:43,617 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_2196. 2024-12-10T10:50:43,618 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_4315. 2024-12-10T10:50:43,618 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_6402. 2024-12-10T10:50:43,618 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_8499. 2024-12-10T10:50:43,618 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_10697. 2024-12-10T10:50:43,619 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_12878. 2024-12-10T10:50:43,619 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_14979. 2024-12-10T10:50:43,619 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_17067. 2024-12-10T10:50:43,619 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_19239. 2024-12-10T10:50:43,620 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_21339. 2024-12-10T10:50:43,620 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_23494. 2024-12-10T10:50:43,620 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_25591. 2024-12-10T10:50:43,620 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_27736. 2024-12-10T10:50:43,621 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_29841. 2024-12-10T10:50:43,621 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_31954. 2024-12-10T10:50:43,621 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_34169. 2024-12-10T10:50:43,622 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_36313. 2024-12-10T10:50:43,622 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_38405. 2024-12-10T10:50:43,622 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_40528. 2024-12-10T10:50:43,623 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_42679. 2024-12-10T10:50:43,623 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_44855. 2024-12-10T10:50:43,623 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_46941. 2024-12-10T10:50:43,623 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_49063. 2024-12-10T10:50:43,624 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_51167. 2024-12-10T10:50:43,624 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_53302. 2024-12-10T10:50:43,624 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_55504. 2024-12-10T10:50:43,625 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_57644. 2024-12-10T10:50:43,625 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_59772. 2024-12-10T10:50:43,625 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_61889. 2024-12-10T10:50:43,625 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_64095. 2024-12-10T10:50:43,626 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_66261. 2024-12-10T10:50:43,626 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_68453. 2024-12-10T10:50:43,626 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_70627. 2024-12-10T10:50:43,627 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_72750. 2024-12-10T10:50:43,627 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_74877. 2024-12-10T10:50:43,627 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_77049. 2024-12-10T10:50:43,627 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_79263. 2024-12-10T10:50:43,628 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_81406. 2024-12-10T10:50:43,628 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_83513. 2024-12-10T10:50:43,628 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_85622. 2024-12-10T10:50:43,628 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_87816. 2024-12-10T10:50:43,629 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_89914. 2024-12-10T10:50:43,629 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_92049. 2024-12-10T10:50:43,629 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_94233. 2024-12-10T10:50:43,630 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_96386. 2024-12-10T10:50:43,630 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_98556. 2024-12-10T10:50:43,630 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_100668. 2024-12-10T10:50:43,631 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_102754. 2024-12-10T10:50:43,631 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_104849. 2024-12-10T10:50:43,631 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_106971. 2024-12-10T10:50:43,632 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_109072. 2024-12-10T10:50:43,632 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_111209. 2024-12-10T10:50:43,632 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_113393. 2024-12-10T10:50:43,633 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_115538. 2024-12-10T10:50:43,633 DEBUG [hfile-prefetch-1733827838340 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d15845c9e82d4e39be52b6842d45b1a5_117611. 2024-12-10T10:50:43,633 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch0/d15845c9e82d4e39be52b6842d45b1a5, with total blocks 56, and data blocks 55 2024-12-10T10:50:43,633 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2285): iterating over 42 entries in the backing map 2024-12-10T10:50:43,633 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 0 2024-12-10T10:50:43,634 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 2196 2024-12-10T10:50:43,634 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 4315 2024-12-10T10:50:43,634 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 6402 2024-12-10T10:50:43,634 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 8499 2024-12-10T10:50:43,634 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 10697 2024-12-10T10:50:43,634 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 12878 2024-12-10T10:50:43,634 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 14979 2024-12-10T10:50:43,634 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 17067 2024-12-10T10:50:43,634 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 19239 2024-12-10T10:50:43,634 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 21339 2024-12-10T10:50:43,634 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 23494 2024-12-10T10:50:43,634 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 25591 2024-12-10T10:50:43,634 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 27736 2024-12-10T10:50:43,634 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 29841 2024-12-10T10:50:43,634 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 31954 2024-12-10T10:50:43,634 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 34169 2024-12-10T10:50:43,634 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 36313 2024-12-10T10:50:43,634 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 38405 2024-12-10T10:50:43,635 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 40528 2024-12-10T10:50:43,635 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 42679 2024-12-10T10:50:43,635 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 44855 2024-12-10T10:50:43,635 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 46941 2024-12-10T10:50:43,635 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 49063 2024-12-10T10:50:43,635 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 51167 2024-12-10T10:50:43,635 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 53302 2024-12-10T10:50:43,635 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 55504 2024-12-10T10:50:43,635 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 57644 2024-12-10T10:50:43,635 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 59772 2024-12-10T10:50:43,635 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 61889 2024-12-10T10:50:43,635 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 64095 2024-12-10T10:50:43,635 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 66261 2024-12-10T10:50:43,635 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 68453 2024-12-10T10:50:43,636 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 70627 2024-12-10T10:50:43,636 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 72750 2024-12-10T10:50:43,636 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 74877 2024-12-10T10:50:43,636 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 77049 2024-12-10T10:50:43,636 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 79263 2024-12-10T10:50:43,636 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 81406 2024-12-10T10:50:43,636 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 83513 2024-12-10T10:50:43,636 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 85622 2024-12-10T10:50:43,636 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 87816 2024-12-10T10:50:43,636 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 89914 2024-12-10T10:50:43,636 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 92049 2024-12-10T10:50:43,636 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 94233 2024-12-10T10:50:43,636 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 96386 2024-12-10T10:50:43,636 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 98556 2024-12-10T10:50:43,636 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 100668 2024-12-10T10:50:43,636 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 102754 2024-12-10T10:50:43,636 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 104849 2024-12-10T10:50:43,636 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 106971 2024-12-10T10:50:43,636 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 109072 2024-12-10T10:50:43,636 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 111209 2024-12-10T10:50:43,636 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch0/d15845c9e82d4e39be52b6842d45b1a5, but only 53 data blocks were cached. Total data blocks for file: 55. Checking for blocks pending cache in cache writer queue. 2024-12-10T10:50:43,637 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch0/d15845c9e82d4e39be52b6842d45b1a5. Will sleep 100ms and try the verification again. 2024-12-10T10:50:43,737 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch0/d15845c9e82d4e39be52b6842d45b1a5, with total blocks 56, and data blocks 55 2024-12-10T10:50:43,737 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2285): iterating over 56 entries in the backing map 2024-12-10T10:50:43,737 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 0 2024-12-10T10:50:43,737 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 2196 2024-12-10T10:50:43,737 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 4315 2024-12-10T10:50:43,737 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 6402 2024-12-10T10:50:43,737 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 8499 2024-12-10T10:50:43,737 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 10697 2024-12-10T10:50:43,737 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 12878 2024-12-10T10:50:43,737 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 14979 2024-12-10T10:50:43,737 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 17067 2024-12-10T10:50:43,738 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 19239 2024-12-10T10:50:43,738 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 21339 2024-12-10T10:50:43,738 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 23494 2024-12-10T10:50:43,738 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 25591 2024-12-10T10:50:43,738 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 27736 2024-12-10T10:50:43,738 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 29841 2024-12-10T10:50:43,738 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 31954 2024-12-10T10:50:43,738 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 34169 2024-12-10T10:50:43,738 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 36313 2024-12-10T10:50:43,738 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 38405 2024-12-10T10:50:43,738 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 40528 2024-12-10T10:50:43,738 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 42679 2024-12-10T10:50:43,738 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 44855 2024-12-10T10:50:43,738 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 46941 2024-12-10T10:50:43,738 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 49063 2024-12-10T10:50:43,738 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 51167 2024-12-10T10:50:43,738 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 53302 2024-12-10T10:50:43,738 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 55504 2024-12-10T10:50:43,738 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 57644 2024-12-10T10:50:43,738 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 59772 2024-12-10T10:50:43,738 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 61889 2024-12-10T10:50:43,738 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 64095 2024-12-10T10:50:43,739 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 66261 2024-12-10T10:50:43,739 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 68453 2024-12-10T10:50:43,739 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 70627 2024-12-10T10:50:43,739 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 72750 2024-12-10T10:50:43,739 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 74877 2024-12-10T10:50:43,739 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 77049 2024-12-10T10:50:43,739 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 79263 2024-12-10T10:50:43,739 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 81406 2024-12-10T10:50:43,739 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 83513 2024-12-10T10:50:43,739 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 85622 2024-12-10T10:50:43,739 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 87816 2024-12-10T10:50:43,739 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 89914 2024-12-10T10:50:43,739 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 92049 2024-12-10T10:50:43,739 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 94233 2024-12-10T10:50:43,739 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 96386 2024-12-10T10:50:43,739 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 98556 2024-12-10T10:50:43,739 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 100668 2024-12-10T10:50:43,739 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 102754 2024-12-10T10:50:43,739 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 104849 2024-12-10T10:50:43,739 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 106971 2024-12-10T10:50:43,739 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 109072 2024-12-10T10:50:43,739 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 111209 2024-12-10T10:50:43,739 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 113393 2024-12-10T10:50:43,739 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 115538 2024-12-10T10:50:43,740 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2293): found block for file d15845c9e82d4e39be52b6842d45b1a5 in the backing map. Acquiring read lock for offset 117611 2024-12-10T10:50:43,740 DEBUG [hfile-prefetch-1733827838340 {}] bucket.BucketCache(2305): File /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch0/d15845c9e82d4e39be52b6842d45b1a5 has now been fully cached. 2024-12-10T10:50:43,740 DEBUG [hfile-prefetch-1733827838340 {}] hfile.PrefetchExecutor(121): Prefetch completed for d15845c9e82d4e39be52b6842d45b1a5 2024-12-10T10:50:44,673 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch1/299160daa1ed4f08b95094e277d652fc, delay=927 ms 2024-12-10T10:50:45,601 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_0. 2024-12-10T10:50:45,602 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_2088. 2024-12-10T10:50:45,602 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_4258. 2024-12-10T10:50:45,603 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_6374. 2024-12-10T10:50:45,603 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_8520. 2024-12-10T10:50:45,603 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_10654. 2024-12-10T10:50:45,604 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_12792. 2024-12-10T10:50:45,604 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_14935. 2024-12-10T10:50:45,604 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_17068. 2024-12-10T10:50:45,604 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_19227. 2024-12-10T10:50:45,604 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_21327. 2024-12-10T10:50:45,605 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_23434. 2024-12-10T10:50:45,605 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_25531. 2024-12-10T10:50:45,605 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_27796. 2024-12-10T10:50:45,606 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_29884. 2024-12-10T10:50:45,606 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_32002. 2024-12-10T10:50:45,606 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_34110. 2024-12-10T10:50:45,606 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_36274. 2024-12-10T10:50:45,607 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_38408. 2024-12-10T10:50:45,607 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_40542. 2024-12-10T10:50:45,607 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_42709. 2024-12-10T10:50:45,608 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_44918. 2024-12-10T10:50:45,608 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_47055. 2024-12-10T10:50:45,608 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_49174. 2024-12-10T10:50:45,608 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_51342. 2024-12-10T10:50:45,609 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_53472. 2024-12-10T10:50:45,609 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_55626. 2024-12-10T10:50:45,609 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_57793. 2024-12-10T10:50:45,609 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_60006. 2024-12-10T10:50:45,610 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_62134. 2024-12-10T10:50:45,610 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_64236. 2024-12-10T10:50:45,610 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_66391. 2024-12-10T10:50:45,611 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_68604. 2024-12-10T10:50:45,611 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_70716. 2024-12-10T10:50:45,611 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_72898. 2024-12-10T10:50:45,611 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_75027. 2024-12-10T10:50:45,612 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_77170. 2024-12-10T10:50:45,612 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_79316. 2024-12-10T10:50:45,612 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_81411. 2024-12-10T10:50:45,612 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_83546. 2024-12-10T10:50:45,613 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_85733. 2024-12-10T10:50:45,613 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_87946. 2024-12-10T10:50:45,613 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_90052. 2024-12-10T10:50:45,613 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_92189. 2024-12-10T10:50:45,614 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_94288. 2024-12-10T10:50:45,614 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_96376. 2024-12-10T10:50:45,614 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_98481. 2024-12-10T10:50:45,614 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_100649. 2024-12-10T10:50:45,615 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_102775. 2024-12-10T10:50:45,615 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_104926. 2024-12-10T10:50:45,615 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_107038. 2024-12-10T10:50:45,615 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_109183. 2024-12-10T10:50:45,616 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_111351. 2024-12-10T10:50:45,616 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_113531. 2024-12-10T10:50:45,616 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_115619. 2024-12-10T10:50:45,617 DEBUG [hfile-prefetch-1733827841606 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 299160daa1ed4f08b95094e277d652fc_116835. 2024-12-10T10:50:45,617 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch1/299160daa1ed4f08b95094e277d652fc, with total blocks 56, and data blocks 55 2024-12-10T10:50:45,617 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2285): iterating over 88 entries in the backing map 2024-12-10T10:50:45,617 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 0 2024-12-10T10:50:45,617 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 2088 2024-12-10T10:50:45,617 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 4258 2024-12-10T10:50:45,617 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 6374 2024-12-10T10:50:45,617 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 8520 2024-12-10T10:50:45,617 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 10654 2024-12-10T10:50:45,617 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 12792 2024-12-10T10:50:45,617 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 14935 2024-12-10T10:50:45,617 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 17068 2024-12-10T10:50:45,617 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 19227 2024-12-10T10:50:45,617 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 21327 2024-12-10T10:50:45,618 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 23434 2024-12-10T10:50:45,618 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 25531 2024-12-10T10:50:45,618 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 27796 2024-12-10T10:50:45,618 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 29884 2024-12-10T10:50:45,618 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 32002 2024-12-10T10:50:45,618 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 34110 2024-12-10T10:50:45,618 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 36274 2024-12-10T10:50:45,618 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 38408 2024-12-10T10:50:45,618 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 40542 2024-12-10T10:50:45,618 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 42709 2024-12-10T10:50:45,618 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 44918 2024-12-10T10:50:45,618 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 47055 2024-12-10T10:50:45,618 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 49174 2024-12-10T10:50:45,618 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 51342 2024-12-10T10:50:45,618 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 53472 2024-12-10T10:50:45,618 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 55626 2024-12-10T10:50:45,618 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 57793 2024-12-10T10:50:45,618 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 60006 2024-12-10T10:50:45,618 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 62134 2024-12-10T10:50:45,618 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 64236 2024-12-10T10:50:45,618 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 66391 2024-12-10T10:50:45,618 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 68604 2024-12-10T10:50:45,618 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 70716 2024-12-10T10:50:45,618 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 72898 2024-12-10T10:50:45,618 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 75027 2024-12-10T10:50:45,618 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 77170 2024-12-10T10:50:45,618 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 79316 2024-12-10T10:50:45,618 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 81411 2024-12-10T10:50:45,618 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 83546 2024-12-10T10:50:45,619 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch1/299160daa1ed4f08b95094e277d652fc, but only 40 data blocks were cached. Total data blocks for file: 55. Checking for blocks pending cache in cache writer queue. 2024-12-10T10:50:45,619 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch1/299160daa1ed4f08b95094e277d652fc. Will sleep 100ms and try the verification again. 2024-12-10T10:50:45,640 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-10T10:50:45,640 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-10T10:50:45,640 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-10T10:50:45,641 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 112, fullycachedFiles size: 1, chunkSize: 10000 2024-12-10T10:50:45,645 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 112, fullycachedFiles size: 1 2024-12-10T10:50:45,645 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 112, fullycachedFiles size: 1, file name: bucket.persistence1733827845640 2024-12-10T10:50:45,645 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-10T10:50:45,719 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch1/299160daa1ed4f08b95094e277d652fc, with total blocks 56, and data blocks 55 2024-12-10T10:50:45,719 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2285): iterating over 112 entries in the backing map 2024-12-10T10:50:45,719 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 0 2024-12-10T10:50:45,719 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 2088 2024-12-10T10:50:45,719 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 4258 2024-12-10T10:50:45,719 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 6374 2024-12-10T10:50:45,719 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 8520 2024-12-10T10:50:45,719 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 10654 2024-12-10T10:50:45,719 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 12792 2024-12-10T10:50:45,719 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 14935 2024-12-10T10:50:45,719 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 17068 2024-12-10T10:50:45,719 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 19227 2024-12-10T10:50:45,719 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 21327 2024-12-10T10:50:45,719 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 23434 2024-12-10T10:50:45,719 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 25531 2024-12-10T10:50:45,719 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 27796 2024-12-10T10:50:45,719 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 29884 2024-12-10T10:50:45,719 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 32002 2024-12-10T10:50:45,719 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 34110 2024-12-10T10:50:45,719 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 36274 2024-12-10T10:50:45,719 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 38408 2024-12-10T10:50:45,720 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 40542 2024-12-10T10:50:45,720 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 42709 2024-12-10T10:50:45,720 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 44918 2024-12-10T10:50:45,720 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 47055 2024-12-10T10:50:45,720 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 49174 2024-12-10T10:50:45,720 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 51342 2024-12-10T10:50:45,720 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 53472 2024-12-10T10:50:45,720 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 55626 2024-12-10T10:50:45,720 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 57793 2024-12-10T10:50:45,720 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 60006 2024-12-10T10:50:45,720 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 62134 2024-12-10T10:50:45,720 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 64236 2024-12-10T10:50:45,720 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 66391 2024-12-10T10:50:45,720 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 68604 2024-12-10T10:50:45,720 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 70716 2024-12-10T10:50:45,720 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 72898 2024-12-10T10:50:45,720 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 75027 2024-12-10T10:50:45,720 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 77170 2024-12-10T10:50:45,720 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 79316 2024-12-10T10:50:45,720 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 81411 2024-12-10T10:50:45,720 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 83546 2024-12-10T10:50:45,720 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 85733 2024-12-10T10:50:45,720 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 87946 2024-12-10T10:50:45,720 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 90052 2024-12-10T10:50:45,720 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 92189 2024-12-10T10:50:45,720 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 94288 2024-12-10T10:50:45,720 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 96376 2024-12-10T10:50:45,720 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 98481 2024-12-10T10:50:45,721 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 100649 2024-12-10T10:50:45,721 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 102775 2024-12-10T10:50:45,721 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 104926 2024-12-10T10:50:45,721 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 107038 2024-12-10T10:50:45,721 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 109183 2024-12-10T10:50:45,721 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 111351 2024-12-10T10:50:45,721 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 113531 2024-12-10T10:50:45,721 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 115619 2024-12-10T10:50:45,721 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2293): found block for file 299160daa1ed4f08b95094e277d652fc in the backing map. Acquiring read lock for offset 116835 2024-12-10T10:50:45,721 DEBUG [hfile-prefetch-1733827841606 {}] bucket.BucketCache(2305): File /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/TestPrefetch1/299160daa1ed4f08b95094e277d652fc has now been fully cached. 2024-12-10T10:50:45,721 DEBUG [hfile-prefetch-1733827841606 {}] hfile.PrefetchExecutor(121): Prefetch completed for 299160daa1ed4f08b95094e277d652fc 2024-12-10T10:50:49,675 INFO [Time-limited test {}] bucket.BucketCache(1738): Disabling cache 2024-12-10T10:50:49,676 INFO [Time-limited test {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/702be8da-b702-3ba1-3bea-f8df079f1088/bucket.persistence 2024-12-10T10:50:49,676 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-10T10:50:49,676 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-10T10:50:49,676 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-10T10:50:49,676 INFO [Time-limited test {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-10T10:50:49,676 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-10T10:50:49,686 DEBUG [Time-limited test {}] bucket.BucketCache(1487): Thread Time-limited test started persisting bucket cache to file 2024-12-10T10:50:49,686 DEBUG [Time-limited test {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-10T10:50:49,686 DEBUG [Time-limited test {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 112, fullycachedFiles size: 2, chunkSize: 10000 2024-12-10T10:50:49,691 DEBUG [Time-limited test {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 112, fullycachedFiles size: 2 2024-12-10T10:50:49,691 DEBUG [Time-limited test {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 112, fullycachedFiles size: 2, file name: bucket.persistence1733827849686 2024-12-10T10:50:49,691 DEBUG [Time-limited test {}] bucket.BucketCache(1509): Thread Time-limited test finished persisting bucket cache to file, renaming 2024-12-10T10:50:49,691 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T10:50:49,692 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T10:50:49,699 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestBucketCachePersister#testPrefetchPersistenceCrash Thread=23 (was 22) Potentially hanging thread: hfile-prefetch-1733827842670 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=292 (was 290) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=143 (was 156), ProcessCount=11 (was 11), AvailableMemoryMB=4568 (was 4577)