2024-12-12 02:07:54,408 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@57bc27f5
2024-12-12 02:07:54,424 main DEBUG Took 0.014381 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging
2024-12-12 02:07:54,425 main DEBUG PluginManager 'Core' found 129 plugins
2024-12-12 02:07:54,425 main DEBUG PluginManager 'Level' found 0 plugins
2024-12-12 02:07:54,426 main DEBUG PluginManager 'Lookup' found 16 plugins
2024-12-12 02:07:54,427 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-12 02:07:54,433 main DEBUG PluginManager 'TypeConverter' found 26 plugins
2024-12-12 02:07:54,443 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-12 02:07:54,445 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-12 02:07:54,445 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-12 02:07:54,445 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-12 02:07:54,446 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-12 02:07:54,446 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-12 02:07:54,447 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-12 02:07:54,447 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-12 02:07:54,448 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-12 02:07:54,448 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-12 02:07:54,449 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-12 02:07:54,449 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-12 02:07:54,449 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-12 02:07:54,450 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-12 02:07:54,450 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-12 02:07:54,450 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-12 02:07:54,451 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-12 02:07:54,451 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-12 02:07:54,451 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-12 02:07:54,451 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-12 02:07:54,452 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-12 02:07:54,452 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-12 02:07:54,452 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-12 02:07:54,453 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-12 02:07:54,453 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-12 02:07:54,453 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger].
2024-12-12 02:07:54,454 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-12 02:07:54,456 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin].
2024-12-12 02:07:54,457 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root})
2024-12-12 02:07:54,458 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout].
2024-12-12 02:07:54,459 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null")
2024-12-12 02:07:54,459 main DEBUG PluginManager 'Converter' found 47 plugins
2024-12-12 02:07:54,466 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender].
2024-12-12 02:07:54,469 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={})
2024-12-12 02:07:54,470 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR
2024-12-12 02:07:54,471 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin].
2024-12-12 02:07:54,471 main DEBUG createAppenders(={Console})
2024-12-12 02:07:54,472 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@57bc27f5 initialized
2024-12-12 02:07:54,472 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@57bc27f5
2024-12-12 02:07:54,472 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@57bc27f5 OK.
2024-12-12 02:07:54,473 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1
2024-12-12 02:07:54,473 main DEBUG OutputStream closed
2024-12-12 02:07:54,473 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true
2024-12-12 02:07:54,474 main DEBUG Appender DefaultConsole-1 stopped with status true
2024-12-12 02:07:54,474 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5a56cdac OK
2024-12-12 02:07:54,542 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6
2024-12-12 02:07:54,544 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger
2024-12-12 02:07:54,545 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector
2024-12-12 02:07:54,546 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=
2024-12-12 02:07:54,546 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory
2024-12-12 02:07:54,547 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter
2024-12-12 02:07:54,547 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper
2024-12-12 02:07:54,548 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j
2024-12-12 02:07:54,548 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl
2024-12-12 02:07:54,549 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans
2024-12-12 02:07:54,549 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase
2024-12-12 02:07:54,550 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop
2024-12-12 02:07:54,550 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers
2024-12-12 02:07:54,551 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices
2024-12-12 02:07:54,551 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig
2024-12-12 02:07:54,551 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel
2024-12-12 02:07:54,551 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore
2024-12-12 02:07:54,552 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console
2024-12-12 02:07:54,554 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-12-12 02:07:54,554 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@1608bcbd) with optional ClassLoader: null
2024-12-12 02:07:54,554 main DEBUG Shutdown hook enabled. Registering a new one.
2024-12-12 02:07:54,555 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@1608bcbd] started OK.
2024-12-12T02:07:54,567 INFO  [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.bucket.TestBucketCachePersister timeout: 13 mins
2024-12-12 02:07:54,570 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED)
2024-12-12 02:07:54,570 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-12-12T02:07:54,770 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81
2024-12-12T02:07:54,794 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestBucketCachePersister#testPrefetchBlockEvictionWhilePrefetchRunning Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=796, ProcessCount=11, AvailableMemoryMB=9005
2024-12-12T02:07:54,875 WARN  [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
2024-12-12T02:07:54,956 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-12T02:07:54,978 INFO  [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/testPrefetchBlockEvictionWhilePrefetchRunning
2024-12-12T02:07:54,986 INFO  [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false
2024-12-12T02:07:54,989 INFO  [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread
2024-12-12T02:07:54,991 INFO  [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 200
2024-12-12T02:07:54,993 INFO  [Thread-5 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file
2024-12-12T02:07:54,993 WARN  [Thread-5 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache.
2024-12-12T02:07:54,998 INFO  [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/testPrefetchBlockEvictionWhilePrefetchRunning, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator
2024-12-12T02:07:55,000 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-12T02:07:55,001 INFO  [Thread-5 {}] bucket.BucketAllocator(343): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size))
2024-12-12T02:07:55,001 INFO  [Thread-5 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/bucket.persistence is complete.
2024-12-12T02:07:55,024 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout
2024-12-12T02:07:55,075 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d272e19cfd1f4ab7981a11c5ac52c17d
2024-12-12T02:07:55,166 WARN  [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-12-12T02:07:55,195 INFO  [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl
2024-12-12T02:07:55,231 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is 230, key is aaaaaaaaaaaaaaaaaaaaaabaaaababbb/pD:qgac0POi5Qtk/-5604394523074612892/DeleteFamilyVersion/seqid=0
2024-12-12T02:07:55,319 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d, delay=905 ms
2024-12-12T02:07:55,350 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,361 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,371 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,382 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,392 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,403 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,413 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,424 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,434 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,445 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,456 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,466 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,477 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,487 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,497 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,508 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,518 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,529 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,539 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,550 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,560 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,571 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,582 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,592 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,603 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,614 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,625 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,636 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,647 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,658 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,669 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,679 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,689 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,700 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,710 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,720 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,731 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,742 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,752 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,763 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,774 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,784 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,795 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,806 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,817 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,827 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,838 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,849 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,860 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,871 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,882 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,892 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,903 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,914 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,925 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,936 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,947 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,958 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,968 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,979 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:55,990 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:56,000 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:56,011 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:56,022 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:56,032 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:56,043 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:56,053 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:56,064 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:56,075 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:56,085 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:56,096 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:56,107 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:56,118 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:56,129 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:56,139 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:56,150 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:56,160 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:56,171 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:56,181 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:56,192 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:56,202 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:56,213 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:56,224 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:56,234 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:56,240 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_0. 
2024-12-12T02:07:56,245 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:56,255 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:56,265 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:56,271 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_2109. 
2024-12-12T02:07:56,272 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_4251. 
2024-12-12T02:07:56,273 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_6436. 
2024-12-12T02:07:56,274 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_8592. 
2024-12-12T02:07:56,274 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_10727. 
2024-12-12T02:07:56,274 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_12814. 
2024-12-12T02:07:56,275 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_14901. 
2024-12-12T02:07:56,275 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_17085. 
2024-12-12T02:07:56,276 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:56,276 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_19205. 
2024-12-12T02:07:56,276 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_21363. 
2024-12-12T02:07:56,277 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_23527. 
2024-12-12T02:07:56,277 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_25743. 
2024-12-12T02:07:56,277 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_27949. 
2024-12-12T02:07:56,278 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_30086. 
2024-12-12T02:07:56,278 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_32246. 
2024-12-12T02:07:56,278 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_34410. 
2024-12-12T02:07:56,279 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_36615. 
2024-12-12T02:07:56,279 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_38853. 
2024-12-12T02:07:56,280 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_40968. 
2024-12-12T02:07:56,280 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_43158. 
2024-12-12T02:07:56,281 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_45322. 
2024-12-12T02:07:56,281 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_47449. 
2024-12-12T02:07:56,282 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_49553. 
2024-12-12T02:07:56,282 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_51677. 
2024-12-12T02:07:56,282 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_53894. 
2024-12-12T02:07:56,283 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_56038. 
2024-12-12T02:07:56,283 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_58153. 
2024-12-12T02:07:56,284 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_60355. 
2024-12-12T02:07:56,284 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_62462. 
2024-12-12T02:07:56,284 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_64579. 
2024-12-12T02:07:56,285 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_66708. 
2024-12-12T02:07:56,285 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_68861. 
2024-12-12T02:07:56,286 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_70957. 
2024-12-12T02:07:56,286 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:56,286 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_73100. 
2024-12-12T02:07:56,287 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_75276. 
2024-12-12T02:07:56,287 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_77410. 
2024-12-12T02:07:56,288 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_79543. 
2024-12-12T02:07:56,288 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_81737. 
2024-12-12T02:07:56,288 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_83860. 
2024-12-12T02:07:56,289 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_85975. 
2024-12-12T02:07:56,289 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_88214. 
2024-12-12T02:07:56,290 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_90347. 
2024-12-12T02:07:56,290 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_92479. 
2024-12-12T02:07:56,291 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_94599. 
2024-12-12T02:07:56,291 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_96788. 
2024-12-12T02:07:56,292 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_98889. 
2024-12-12T02:07:56,292 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_101056. 
2024-12-12T02:07:56,293 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_103145. 
2024-12-12T02:07:56,293 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_105313. 
2024-12-12T02:07:56,293 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_107462. 
2024-12-12T02:07:56,294 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_109659. 
2024-12-12T02:07:56,294 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_111753. 
2024-12-12T02:07:56,295 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_113864. 
2024-12-12T02:07:56,295 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_116079. 
2024-12-12T02:07:56,296 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d272e19cfd1f4ab7981a11c5ac52c17d_117795. 
2024-12-12T02:07:56,296 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d, with total blocks 56, and data blocks 55
2024-12-12T02:07:56,296 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:56,297 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map
2024-12-12T02:07:56,298 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d, but only 0 data blocks were cached. Total data blocks for file: 55. Checking for blocks pending cache in cache writer queue.
2024-12-12T02:07:56,298 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d. Will sleep 100ms and try the verification again.
2024-12-12T02:07:56,306 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:56,317 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:56,327 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:56,337 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:56,347 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:56,358 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:56,360 DEBUG [Time-limited test {}] bucket.BucketCache(856): removed key d272e19cfd1f4ab7981a11c5ac52c17d_0 from back map with offset lock 525312 in the evict process
2024-12-12T02:07:56,360 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(182): Attempted eviction for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d. Succeeded? true
2024-12-12T02:07:56,370 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:56,381 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:56,391 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:56,392 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk
2024-12-12T02:07:56,393 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file
2024-12-12T02:07:56,393 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format.
2024-12-12T02:07:56,394 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 2, fullycachedFiles size: 0, chunkSize: 10000
2024-12-12T02:07:56,399 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d, with total blocks 56, and data blocks 55
2024-12-12T02:07:56,399 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2285): iterating over 2 entries in the backing map
2024-12-12T02:07:56,399 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 6436
2024-12-12T02:07:56,399 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 10727
2024-12-12T02:07:56,400 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d, but only 2 data blocks were cached. Total data blocks for file: 55. Checking for blocks pending cache in cache writer queue.
2024-12-12T02:07:56,400 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d. Will sleep 100ms and try the verification again.
2024-12-12T02:07:56,402 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:56,412 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:56,422 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:56,433 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:56,443 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:56,454 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:56,464 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:56,472 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 55, fullycachedFiles size: 0
2024-12-12T02:07:56,473 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 55, fullycachedFiles size: 0, file name: bucket.persistence1733969276393
2024-12-12T02:07:56,473 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming
2024-12-12T02:07:56,474 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:56,484 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:56,495 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:56,500 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d, with total blocks 56, and data blocks 55
2024-12-12T02:07:56,501 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2285): iterating over 55 entries in the backing map
2024-12-12T02:07:56,501 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 2109
2024-12-12T02:07:56,501 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 4251
2024-12-12T02:07:56,501 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 6436
2024-12-12T02:07:56,501 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 8592
2024-12-12T02:07:56,501 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 10727
2024-12-12T02:07:56,502 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 12814
2024-12-12T02:07:56,502 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 14901
2024-12-12T02:07:56,502 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 17085
2024-12-12T02:07:56,502 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 19205
2024-12-12T02:07:56,502 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 21363
2024-12-12T02:07:56,503 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 23527
2024-12-12T02:07:56,503 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 25743
2024-12-12T02:07:56,503 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 27949
2024-12-12T02:07:56,503 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 30086
2024-12-12T02:07:56,503 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 32246
2024-12-12T02:07:56,503 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 34410
2024-12-12T02:07:56,504 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 36615
2024-12-12T02:07:56,504 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 38853
2024-12-12T02:07:56,504 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 40968
2024-12-12T02:07:56,504 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 43158
2024-12-12T02:07:56,504 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 45322
2024-12-12T02:07:56,505 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 47449
2024-12-12T02:07:56,505 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 49553
2024-12-12T02:07:56,505 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 51677
2024-12-12T02:07:56,505 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d is still running.
2024-12-12T02:07:56,505 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 53894
2024-12-12T02:07:56,505 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 56038
2024-12-12T02:07:56,506 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 58153
2024-12-12T02:07:56,506 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 60355
2024-12-12T02:07:56,506 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 62462
2024-12-12T02:07:56,506 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 64579
2024-12-12T02:07:56,507 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 66708
2024-12-12T02:07:56,507 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 68861
2024-12-12T02:07:56,507 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 70957
2024-12-12T02:07:56,508 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 73100
2024-12-12T02:07:56,508 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 75276
2024-12-12T02:07:56,508 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 77410
2024-12-12T02:07:56,508 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 79543
2024-12-12T02:07:56,509 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 81737
2024-12-12T02:07:56,509 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 83860
2024-12-12T02:07:56,509 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 85975
2024-12-12T02:07:56,509 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 88214
2024-12-12T02:07:56,510 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 90347
2024-12-12T02:07:56,510 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 92479
2024-12-12T02:07:56,510 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 94599
2024-12-12T02:07:56,510 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 96788
2024-12-12T02:07:56,510 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 98889
2024-12-12T02:07:56,511 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 101056
2024-12-12T02:07:56,511 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 103145
2024-12-12T02:07:56,511 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 105313
2024-12-12T02:07:56,511 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 107462
2024-12-12T02:07:56,511 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 109659
2024-12-12T02:07:56,511 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 111753
2024-12-12T02:07:56,512 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 113864
2024-12-12T02:07:56,512 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 116079
2024-12-12T02:07:56,512 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file d272e19cfd1f4ab7981a11c5ac52c17d in the backing map. Acquiring read lock for offset 117795
2024-12-12T02:07:56,512 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d, but only 54 data blocks were cached. Total data blocks for file: 55. Checking for blocks pending cache in cache writer queue.
2024-12-12T02:07:56,512 INFO  [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2331): We found only 54 data blocks cached from a total of 55 for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/d272e19cfd1f4ab7981a11c5ac52c17d, but no blocks pending caching. Maybe cache is full or evictions happened concurrently to cache prefetch.
2024-12-12T02:07:56,513 DEBUG [hfile-prefetch-1733969275320 {}] hfile.PrefetchExecutor(121): Prefetch completed for d272e19cfd1f4ab7981a11c5ac52c17d
2024-12-12T02:07:56,515 INFO  [Time-limited test {}] bucket.BucketCache(1738): Disabling cache
2024-12-12T02:07:56,516 INFO  [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false
2024-12-12T02:07:56,516 INFO  [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false
2024-12-12T02:07:56,516 INFO  [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false
2024-12-12T02:07:56,517 INFO  [Time-limited test {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/bucket.persistence
2024-12-12T02:07:56,517 INFO  [Time-limited test {}] bucket.BucketCache(1768): Shutting down cache persister thread.
2024-12-12T02:07:56,517 WARN  [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread.
java.lang.InterruptedException: sleep interrupted
	at java.lang.Thread.sleep(Native Method) ~[?:?]
	at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?]
2024-12-12T02:07:56,527 DEBUG [Time-limited test {}] bucket.BucketCache(1487): Thread Time-limited test started persisting bucket cache to file
2024-12-12T02:07:56,527 DEBUG [Time-limited test {}] bucket.BucketCache(1494): Persist in new chunked persistence format.
2024-12-12T02:07:56,528 DEBUG [Time-limited test {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 55, fullycachedFiles size: 0, chunkSize: 10000
2024-12-12T02:07:56,531 DEBUG [Time-limited test {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 55, fullycachedFiles size: 0
2024-12-12T02:07:56,531 DEBUG [Time-limited test {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 55, fullycachedFiles size: 0, file name: bucket.persistence1733969276527
2024-12-12T02:07:56,532 DEBUG [Time-limited test {}] bucket.BucketCache(1509): Thread Time-limited test finished persisting bucket cache to file, renaming
2024-12-12T02:07:56,542 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-12T02:07:56,542 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-12T02:07:56,547 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-12T02:07:56,554 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestBucketCachePersister#testPrefetchBlockEvictionWhilePrefetchRunning Thread=20 (was 12)
Potentially hanging thread: ForkJoinPool.commonPool-worker-3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: hfile-prefetch-1733969275320
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HBase-Metrics2-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: weak-ref-cleaner-strictcontextstorage
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool.commonPool-worker-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: ForkJoinPool.commonPool-worker-2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: Timer for 'HBase' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)
 - Thread LEAK? -, OpenFileDescriptor=288 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=796 (was 796), ProcessCount=11 (was 11), AvailableMemoryMB=8958 (was 9005)
2024-12-12T02:07:56,559 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestBucketCachePersister#testPrefetchPersistenceCrashNegative Thread=20, OpenFileDescriptor=288, MaxFileDescriptor=1048576, SystemLoadAverage=796, ProcessCount=11, AvailableMemoryMB=8957
2024-12-12T02:07:56,559 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-12T02:07:56,562 INFO  [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/testPrefetchPersistenceCrashNegative
2024-12-12T02:07:56,562 INFO  [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false
2024-12-12T02:07:56,562 INFO  [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread
2024-12-12T02:07:56,562 INFO  [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 9223372036854775807
2024-12-12T02:07:56,563 INFO  [Thread-13 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file
2024-12-12T02:07:56,563 INFO  [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/testPrefetchPersistenceCrashNegative, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator
2024-12-12T02:07:56,563 WARN  [Thread-13 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache.
2024-12-12T02:07:56,563 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-12T02:07:56,564 INFO  [Thread-13 {}] bucket.BucketAllocator(343): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size))
2024-12-12T02:07:56,564 INFO  [Thread-13 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/bucket.persistence is complete.
2024-12-12T02:07:56,571 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 61df550cd6f24f67a2a11c119bbe83be
2024-12-12T02:07:56,587 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch2/61df550cd6f24f67a2a11c119bbe83be is 251, key is aaaaaaaaaaaaaaaaaaaaaabaaababaab/3uAvhn:x5TPKJTZcL/1002799326980634900/DeleteFamilyVersion/seqid=0
2024-12-12T02:07:56,589 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch2/61df550cd6f24f67a2a11c119bbe83be, delay=977 ms
2024-12-12T02:07:57,570 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_0. 
2024-12-12T02:07:57,573 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_2154. 
2024-12-12T02:07:57,574 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_4312. 
2024-12-12T02:07:57,575 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_6490. 
2024-12-12T02:07:57,575 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_8650. 
2024-12-12T02:07:57,576 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_10792. 
2024-12-12T02:07:57,576 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_12890. 
2024-12-12T02:07:57,576 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_15005. 
2024-12-12T02:07:57,577 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_17157. 
2024-12-12T02:07:57,577 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_19315. 
2024-12-12T02:07:57,577 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_21428. 
2024-12-12T02:07:57,577 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_23592. 
2024-12-12T02:07:57,578 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_25750. 
2024-12-12T02:07:57,578 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_27909. 
2024-12-12T02:07:57,578 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_30051. 
2024-12-12T02:07:57,579 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_32192. 
2024-12-12T02:07:57,579 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_34287. 
2024-12-12T02:07:57,579 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_36401. 
2024-12-12T02:07:57,579 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_38564. 
2024-12-12T02:07:57,580 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_40750. 
2024-12-12T02:07:57,580 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_42851. 
2024-12-12T02:07:57,580 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_44991. 
2024-12-12T02:07:57,581 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_47174. 
2024-12-12T02:07:57,581 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_49260. 
2024-12-12T02:07:57,581 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_51431. 
2024-12-12T02:07:57,581 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_53590. 
2024-12-12T02:07:57,582 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_55705. 
2024-12-12T02:07:57,582 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_57886. 
2024-12-12T02:07:57,582 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_60043. 
2024-12-12T02:07:57,583 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_62242. 
2024-12-12T02:07:57,583 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_64342. 
2024-12-12T02:07:57,583 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_66439. 
2024-12-12T02:07:57,583 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_68542. 
2024-12-12T02:07:57,584 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_70656. 
2024-12-12T02:07:57,584 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_72773. 
2024-12-12T02:07:57,584 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_74924. 
2024-12-12T02:07:57,584 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_77103. 
2024-12-12T02:07:57,585 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_79201. 
2024-12-12T02:07:57,585 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_81386. 
2024-12-12T02:07:57,585 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_83494. 
2024-12-12T02:07:57,585 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_85691. 
2024-12-12T02:07:57,586 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_87829. 
2024-12-12T02:07:57,586 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_89999. 
2024-12-12T02:07:57,586 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_92124. 
2024-12-12T02:07:57,586 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_94301. 
2024-12-12T02:07:57,587 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_96500. 
2024-12-12T02:07:57,587 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_98635. 
2024-12-12T02:07:57,587 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_100784. 
2024-12-12T02:07:57,587 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_102909. 
2024-12-12T02:07:57,588 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_105000. 
2024-12-12T02:07:57,588 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_107159. 
2024-12-12T02:07:57,588 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_109346. 
2024-12-12T02:07:57,588 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_111497. 
2024-12-12T02:07:57,588 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_113626. 
2024-12-12T02:07:57,589 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_115865. 
2024-12-12T02:07:57,589 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_117998. 
2024-12-12T02:07:57,589 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61df550cd6f24f67a2a11c119bbe83be_119245. 
2024-12-12T02:07:57,589 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch2/61df550cd6f24f67a2a11c119bbe83be, with total blocks 57, and data blocks 56
2024-12-12T02:07:57,589 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map
2024-12-12T02:07:57,590 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch2/61df550cd6f24f67a2a11c119bbe83be, but only 0 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue.
2024-12-12T02:07:57,590 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch2/61df550cd6f24f67a2a11c119bbe83be. Will sleep 100ms and try the verification again.
2024-12-12T02:07:57,690 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch2/61df550cd6f24f67a2a11c119bbe83be, with total blocks 57, and data blocks 56
2024-12-12T02:07:57,690 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2285): iterating over 2 entries in the backing map
2024-12-12T02:07:57,691 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 0
2024-12-12T02:07:57,691 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 4312
2024-12-12T02:07:57,691 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch2/61df550cd6f24f67a2a11c119bbe83be, but only 2 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue.
2024-12-12T02:07:57,692 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch2/61df550cd6f24f67a2a11c119bbe83be. Will sleep 100ms and try the verification again.
2024-12-12T02:07:57,792 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch2/61df550cd6f24f67a2a11c119bbe83be, with total blocks 57, and data blocks 56
2024-12-12T02:07:57,792 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2285): iterating over 57 entries in the backing map
2024-12-12T02:07:57,792 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 0
2024-12-12T02:07:57,793 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 2154
2024-12-12T02:07:57,793 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 4312
2024-12-12T02:07:57,793 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 6490
2024-12-12T02:07:57,793 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 8650
2024-12-12T02:07:57,793 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 10792
2024-12-12T02:07:57,794 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 12890
2024-12-12T02:07:57,794 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 15005
2024-12-12T02:07:57,794 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 17157
2024-12-12T02:07:57,794 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 19315
2024-12-12T02:07:57,794 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 21428
2024-12-12T02:07:57,794 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 23592
2024-12-12T02:07:57,795 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 25750
2024-12-12T02:07:57,795 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 27909
2024-12-12T02:07:57,795 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 30051
2024-12-12T02:07:57,795 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 32192
2024-12-12T02:07:57,796 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 34287
2024-12-12T02:07:57,796 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 36401
2024-12-12T02:07:57,796 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 38564
2024-12-12T02:07:57,796 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 40750
2024-12-12T02:07:57,796 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 42851
2024-12-12T02:07:57,796 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 44991
2024-12-12T02:07:57,797 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 47174
2024-12-12T02:07:57,797 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 49260
2024-12-12T02:07:57,797 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 51431
2024-12-12T02:07:57,797 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 53590
2024-12-12T02:07:57,797 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 55705
2024-12-12T02:07:57,798 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 57886
2024-12-12T02:07:57,798 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 60043
2024-12-12T02:07:57,798 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 62242
2024-12-12T02:07:57,798 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 64342
2024-12-12T02:07:57,799 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 66439
2024-12-12T02:07:57,799 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 68542
2024-12-12T02:07:57,799 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 70656
2024-12-12T02:07:57,799 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 72773
2024-12-12T02:07:57,800 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 74924
2024-12-12T02:07:57,800 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 77103
2024-12-12T02:07:57,800 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 79201
2024-12-12T02:07:57,800 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 81386
2024-12-12T02:07:57,800 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 83494
2024-12-12T02:07:57,801 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 85691
2024-12-12T02:07:57,801 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 87829
2024-12-12T02:07:57,801 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 89999
2024-12-12T02:07:57,801 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 92124
2024-12-12T02:07:57,801 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 94301
2024-12-12T02:07:57,802 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 96500
2024-12-12T02:07:57,802 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 98635
2024-12-12T02:07:57,802 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 100784
2024-12-12T02:07:57,802 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 102909
2024-12-12T02:07:57,802 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 105000
2024-12-12T02:07:57,802 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 107159
2024-12-12T02:07:57,803 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 109346
2024-12-12T02:07:57,803 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 111497
2024-12-12T02:07:57,803 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 113626
2024-12-12T02:07:57,803 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 115865
2024-12-12T02:07:57,803 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 117998
2024-12-12T02:07:57,803 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 61df550cd6f24f67a2a11c119bbe83be in the backing map. Acquiring read lock for offset 119245
2024-12-12T02:07:57,803 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2305): File /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch2/61df550cd6f24f67a2a11c119bbe83be has now been fully cached.
2024-12-12T02:07:57,804 DEBUG [hfile-prefetch-1733969275320 {}] hfile.PrefetchExecutor(121): Prefetch completed for 61df550cd6f24f67a2a11c119bbe83be
2024-12-12T02:07:58,594 INFO  [Time-limited test {}] bucket.BucketCache(1738): Disabling cache
2024-12-12T02:07:58,594 INFO  [Time-limited test {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/bucket.persistence
2024-12-12T02:07:58,594 INFO  [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false
2024-12-12T02:07:58,594 INFO  [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false
2024-12-12T02:07:58,594 INFO  [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false
2024-12-12T02:07:58,595 INFO  [Time-limited test {}] bucket.BucketCache(1768): Shutting down cache persister thread.
2024-12-12T02:07:58,595 WARN  [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread.
java.lang.InterruptedException: sleep interrupted
	at java.lang.Thread.sleep(Native Method) ~[?:?]
	at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?]
2024-12-12T02:07:58,605 DEBUG [Time-limited test {}] bucket.BucketCache(1487): Thread Time-limited test started persisting bucket cache to file
2024-12-12T02:07:58,605 DEBUG [Time-limited test {}] bucket.BucketCache(1494): Persist in new chunked persistence format.
2024-12-12T02:07:58,606 DEBUG [Time-limited test {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 57, fullycachedFiles size: 1, chunkSize: 10000
2024-12-12T02:07:58,613 DEBUG [Time-limited test {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 57, fullycachedFiles size: 1
2024-12-12T02:07:58,613 DEBUG [Time-limited test {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 57, fullycachedFiles size: 1, file name: bucket.persistence1733969278605
2024-12-12T02:07:58,614 DEBUG [Time-limited test {}] bucket.BucketCache(1509): Thread Time-limited test finished persisting bucket cache to file, renaming
2024-12-12T02:07:58,614 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-12T02:07:58,615 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-12T02:07:58,621 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestBucketCachePersister#testPrefetchPersistenceCrashNegative Thread=21 (was 20)
Potentially hanging thread: hfile-prefetch-1733969276590
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=289 (was 288) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=796 (was 796), ProcessCount=11 (was 11), AvailableMemoryMB=8957 (was 8957)
2024-12-12T02:07:58,626 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestBucketCachePersister#testPrefetchListUponBlockEviction Thread=21, OpenFileDescriptor=289, MaxFileDescriptor=1048576, SystemLoadAverage=796, ProcessCount=11, AvailableMemoryMB=8957
2024-12-12T02:07:58,627 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-12T02:07:58,630 INFO  [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/testPrefetchListUponBlockEviction
2024-12-12T02:07:58,631 INFO  [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false
2024-12-12T02:07:58,631 INFO  [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread
2024-12-12T02:07:58,631 INFO  [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 200
2024-12-12T02:07:58,632 INFO  [Thread-18 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file
2024-12-12T02:07:58,632 INFO  [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/testPrefetchListUponBlockEviction, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator
2024-12-12T02:07:58,632 WARN  [Thread-18 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache.
2024-12-12T02:07:58,632 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-12T02:07:58,632 INFO  [Thread-18 {}] bucket.BucketAllocator(343): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size))
2024-12-12T02:07:58,632 INFO  [Thread-18 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/bucket.persistence is complete.
2024-12-12T02:07:58,640 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a442bc4c51be455fa995202e53fe317d
2024-12-12T02:07:58,648 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/a442bc4c51be455fa995202e53fe317d is 227, key is aaaaaaaaaaaaaaaaaaaaaaaaaabbbaaa/LwywO04xOJS/44597578451917423/Put/seqid=0
2024-12-12T02:07:58,650 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/a442bc4c51be455fa995202e53fe317d, delay=918 ms
2024-12-12T02:07:58,832 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk
2024-12-12T02:07:58,832 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file
2024-12-12T02:07:58,832 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format.
2024-12-12T02:07:58,833 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000
2024-12-12T02:07:58,840 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0
2024-12-12T02:07:58,841 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733969278832
2024-12-12T02:07:58,841 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming
2024-12-12T02:07:59,572 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_0. 
2024-12-12T02:07:59,574 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_2134. 
2024-12-12T02:07:59,575 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_4258. 
2024-12-12T02:07:59,575 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_6417. 
2024-12-12T02:07:59,576 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_8514. 
2024-12-12T02:07:59,576 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_10682. 
2024-12-12T02:07:59,577 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_12802. 
2024-12-12T02:07:59,577 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_14939. 
2024-12-12T02:07:59,577 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_17171. 
2024-12-12T02:07:59,578 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_19304. 
2024-12-12T02:07:59,578 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_21513. 
2024-12-12T02:07:59,579 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_23621. 
2024-12-12T02:07:59,579 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_25706. 
2024-12-12T02:07:59,579 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_27791. 
2024-12-12T02:07:59,579 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_30029. 
2024-12-12T02:07:59,580 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_32181. 
2024-12-12T02:07:59,580 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_34400. 
2024-12-12T02:07:59,580 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_36511. 
2024-12-12T02:07:59,580 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_38674. 
2024-12-12T02:07:59,580 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_40861. 
2024-12-12T02:07:59,581 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_43032. 
2024-12-12T02:07:59,581 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_45195. 
2024-12-12T02:07:59,581 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_47348. 
2024-12-12T02:07:59,581 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_49444. 
2024-12-12T02:07:59,582 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_51567. 
2024-12-12T02:07:59,582 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_53669. 
2024-12-12T02:07:59,582 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_55768. 
2024-12-12T02:07:59,582 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_57935. 
2024-12-12T02:07:59,582 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_60142. 
2024-12-12T02:07:59,583 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_62336. 
2024-12-12T02:07:59,583 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_64432. 
2024-12-12T02:07:59,583 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_66595. 
2024-12-12T02:07:59,583 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_68684. 
2024-12-12T02:07:59,583 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_70792. 
2024-12-12T02:07:59,583 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_73005. 
2024-12-12T02:07:59,584 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_75113. 
2024-12-12T02:07:59,584 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_77301. 
2024-12-12T02:07:59,584 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_79432. 
2024-12-12T02:07:59,584 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_81586. 
2024-12-12T02:07:59,585 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_83672. 
2024-12-12T02:07:59,585 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_85767. 
2024-12-12T02:07:59,585 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_87944. 
2024-12-12T02:07:59,585 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_90070. 
2024-12-12T02:07:59,585 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_92256. 
2024-12-12T02:07:59,586 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_94452. 
2024-12-12T02:07:59,586 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_96575. 
2024-12-12T02:07:59,586 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_98678. 
2024-12-12T02:07:59,586 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_100826. 
2024-12-12T02:07:59,587 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_102922. 
2024-12-12T02:07:59,587 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_105013. 
2024-12-12T02:07:59,587 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_107147. 
2024-12-12T02:07:59,587 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_109270. 
2024-12-12T02:07:59,588 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_111361. 
2024-12-12T02:07:59,588 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_113482. 
2024-12-12T02:07:59,588 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_115581. 
2024-12-12T02:07:59,588 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_117732. 
2024-12-12T02:07:59,588 DEBUG [hfile-prefetch-1733969276590 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a442bc4c51be455fa995202e53fe317d_118937. 
2024-12-12T02:07:59,589 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/a442bc4c51be455fa995202e53fe317d, with total blocks 57, and data blocks 56
2024-12-12T02:07:59,589 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map
2024-12-12T02:07:59,589 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/a442bc4c51be455fa995202e53fe317d, but only 0 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue.
2024-12-12T02:07:59,589 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/a442bc4c51be455fa995202e53fe317d. Will sleep 100ms and try the verification again.
2024-12-12T02:07:59,689 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/a442bc4c51be455fa995202e53fe317d, with total blocks 57, and data blocks 56
2024-12-12T02:07:59,689 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2285): iterating over 2 entries in the backing map
2024-12-12T02:07:59,689 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 0
2024-12-12T02:07:59,689 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 2134
2024-12-12T02:07:59,689 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/a442bc4c51be455fa995202e53fe317d, but only 2 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue.
2024-12-12T02:07:59,690 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/a442bc4c51be455fa995202e53fe317d. Will sleep 100ms and try the verification again.
2024-12-12T02:07:59,790 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/a442bc4c51be455fa995202e53fe317d, with total blocks 57, and data blocks 56
2024-12-12T02:07:59,790 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2285): iterating over 57 entries in the backing map
2024-12-12T02:07:59,790 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 0
2024-12-12T02:07:59,790 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 2134
2024-12-12T02:07:59,790 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 4258
2024-12-12T02:07:59,790 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 6417
2024-12-12T02:07:59,790 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 8514
2024-12-12T02:07:59,791 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 10682
2024-12-12T02:07:59,791 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 12802
2024-12-12T02:07:59,791 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 14939
2024-12-12T02:07:59,791 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 17171
2024-12-12T02:07:59,791 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 19304
2024-12-12T02:07:59,791 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 21513
2024-12-12T02:07:59,791 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 23621
2024-12-12T02:07:59,791 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 25706
2024-12-12T02:07:59,791 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 27791
2024-12-12T02:07:59,792 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 30029
2024-12-12T02:07:59,792 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 32181
2024-12-12T02:07:59,792 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 34400
2024-12-12T02:07:59,792 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 36511
2024-12-12T02:07:59,792 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 38674
2024-12-12T02:07:59,792 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 40861
2024-12-12T02:07:59,792 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 43032
2024-12-12T02:07:59,792 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 45195
2024-12-12T02:07:59,792 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 47348
2024-12-12T02:07:59,792 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 49444
2024-12-12T02:07:59,793 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 51567
2024-12-12T02:07:59,793 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 53669
2024-12-12T02:07:59,793 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 55768
2024-12-12T02:07:59,793 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 57935
2024-12-12T02:07:59,793 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 60142
2024-12-12T02:07:59,793 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 62336
2024-12-12T02:07:59,793 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 64432
2024-12-12T02:07:59,793 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 66595
2024-12-12T02:07:59,793 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 68684
2024-12-12T02:07:59,794 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 70792
2024-12-12T02:07:59,794 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 73005
2024-12-12T02:07:59,794 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 75113
2024-12-12T02:07:59,794 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 77301
2024-12-12T02:07:59,794 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 79432
2024-12-12T02:07:59,794 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 81586
2024-12-12T02:07:59,794 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 83672
2024-12-12T02:07:59,794 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 85767
2024-12-12T02:07:59,794 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 87944
2024-12-12T02:07:59,794 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 90070
2024-12-12T02:07:59,795 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 92256
2024-12-12T02:07:59,795 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 94452
2024-12-12T02:07:59,795 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 96575
2024-12-12T02:07:59,795 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 98678
2024-12-12T02:07:59,795 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 100826
2024-12-12T02:07:59,795 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 102922
2024-12-12T02:07:59,795 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 105013
2024-12-12T02:07:59,795 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 107147
2024-12-12T02:07:59,796 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 109270
2024-12-12T02:07:59,796 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 111361
2024-12-12T02:07:59,796 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 113482
2024-12-12T02:07:59,796 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 115581
2024-12-12T02:07:59,796 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 117732
2024-12-12T02:07:59,796 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2293): found block for file a442bc4c51be455fa995202e53fe317d in the backing map. Acquiring read lock for offset 118937
2024-12-12T02:07:59,796 DEBUG [hfile-prefetch-1733969276590 {}] bucket.BucketCache(2305): File /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch3/a442bc4c51be455fa995202e53fe317d has now been fully cached.
2024-12-12T02:07:59,797 DEBUG [hfile-prefetch-1733969276590 {}] hfile.PrefetchExecutor(121): Prefetch completed for a442bc4c51be455fa995202e53fe317d
2024-12-12T02:07:59,842 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk
2024-12-12T02:07:59,842 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file
2024-12-12T02:07:59,842 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format.
2024-12-12T02:07:59,842 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 57, fullycachedFiles size: 1, chunkSize: 10000
2024-12-12T02:07:59,848 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 57, fullycachedFiles size: 1
2024-12-12T02:07:59,848 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 57, fullycachedFiles size: 1, file name: bucket.persistence1733969279842
2024-12-12T02:07:59,848 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming
2024-12-12T02:08:00,653 DEBUG [Time-limited test {}] bucket.BucketCache(856): removed key a442bc4c51be455fa995202e53fe317d_27791 from back map with offset lock 399360 in the evict process
2024-12-12T02:08:00,654 DEBUG [Time-limited test {}] bucket.BucketCache(768): Removing file a442bc4c51be455fa995202e53fe317d for region 38cf2fb7-8fb6-6119-ce82-dac51eb8fd81
2024-12-12T02:08:00,655 INFO  [Time-limited test {}] bucket.BucketCache(1738): Disabling cache
2024-12-12T02:08:00,656 INFO  [Time-limited test {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/bucket.persistence
2024-12-12T02:08:00,656 INFO  [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false
2024-12-12T02:08:00,656 INFO  [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false
2024-12-12T02:08:00,656 INFO  [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false
2024-12-12T02:08:00,657 INFO  [Time-limited test {}] bucket.BucketCache(1768): Shutting down cache persister thread.
2024-12-12T02:08:00,657 WARN  [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread.
java.lang.InterruptedException: sleep interrupted
	at java.lang.Thread.sleep(Native Method) ~[?:?]
	at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?]
2024-12-12T02:08:00,667 DEBUG [Time-limited test {}] bucket.BucketCache(1487): Thread Time-limited test started persisting bucket cache to file
2024-12-12T02:08:00,667 DEBUG [Time-limited test {}] bucket.BucketCache(1494): Persist in new chunked persistence format.
2024-12-12T02:08:00,668 DEBUG [Time-limited test {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 56, fullycachedFiles size: 0, chunkSize: 10000
2024-12-12T02:08:00,673 DEBUG [Time-limited test {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 56, fullycachedFiles size: 0
2024-12-12T02:08:00,673 DEBUG [Time-limited test {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 56, fullycachedFiles size: 0, file name: bucket.persistence1733969280667
2024-12-12T02:08:00,674 DEBUG [Time-limited test {}] bucket.BucketCache(1509): Thread Time-limited test finished persisting bucket cache to file, renaming
2024-12-12T02:08:00,691 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-12T02:08:00,693 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-12T02:08:00,701 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestBucketCachePersister#testPrefetchListUponBlockEviction Thread=22 (was 21)
Potentially hanging thread: hfile-prefetch-1733969278650
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=290 (was 289) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=772 (was 796), ProcessCount=11 (was 11), AvailableMemoryMB=8955 (was 8957)
2024-12-12T02:08:00,706 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestBucketCachePersister#testPrefetchPersistenceCrash Thread=22, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=772, ProcessCount=11, AvailableMemoryMB=8955
2024-12-12T02:08:00,707 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-12T02:08:00,709 INFO  [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/testPrefetchPersistenceCrash
2024-12-12T02:08:00,709 INFO  [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false
2024-12-12T02:08:00,710 INFO  [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread
2024-12-12T02:08:00,710 INFO  [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 3000
2024-12-12T02:08:00,710 INFO  [Thread-25 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file
2024-12-12T02:08:00,710 WARN  [Thread-25 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache.
2024-12-12T02:08:00,710 INFO  [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/testPrefetchPersistenceCrash, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator
2024-12-12T02:08:00,711 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-12T02:08:00,711 INFO  [Thread-25 {}] bucket.BucketAllocator(343): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size))
2024-12-12T02:08:00,711 INFO  [Thread-25 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/bucket.persistence is complete.
2024-12-12T02:08:00,718 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 0379d42162114cd387240baaf64574a0
2024-12-12T02:08:00,726 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch0/0379d42162114cd387240baaf64574a0 is 225, key is aaaaaaaaaaaaaaaaaaaaaaaaaaabbaaa/40fIR4:/-5599243181000906659/Put/seqid=0
2024-12-12T02:08:00,731 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8ed59a40e9a641a2ba1239db0aabde0a
2024-12-12T02:08:00,738 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch1/8ed59a40e9a641a2ba1239db0aabde0a is 238, key is aaaaaaaaaaaaaaaaaaaaaaababbaaaba/L:KsUlE/6844337893014591499/Put/seqid=0
2024-12-12T02:08:00,740 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch0/0379d42162114cd387240baaf64574a0, delay=923 ms
2024-12-12T02:08:01,666 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_0. 
2024-12-12T02:08:01,668 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_2119. 
2024-12-12T02:08:01,669 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_4243. 
2024-12-12T02:08:01,670 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_6384. 
2024-12-12T02:08:01,671 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_8508. 
2024-12-12T02:08:01,672 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_10599. 
2024-12-12T02:08:01,674 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_12686. 
2024-12-12T02:08:01,674 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_14833. 
2024-12-12T02:08:01,675 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_16922. 
2024-12-12T02:08:01,675 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_19036. 
2024-12-12T02:08:01,675 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_21230. 
2024-12-12T02:08:01,675 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_23332. 
2024-12-12T02:08:01,676 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_25536. 
2024-12-12T02:08:01,676 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_27660. 
2024-12-12T02:08:01,676 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_29749. 
2024-12-12T02:08:01,676 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_31866. 
2024-12-12T02:08:01,677 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_33968. 
2024-12-12T02:08:01,677 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_36124. 
2024-12-12T02:08:01,677 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_38301. 
2024-12-12T02:08:01,677 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_40437. 
2024-12-12T02:08:01,678 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_42541. 
2024-12-12T02:08:01,678 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_44643. 
2024-12-12T02:08:01,678 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_46814. 
2024-12-12T02:08:01,678 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_48969. 
2024-12-12T02:08:01,678 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_51073. 
2024-12-12T02:08:01,679 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_53231. 
2024-12-12T02:08:01,679 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_55363. 
2024-12-12T02:08:01,679 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_57498. 
2024-12-12T02:08:01,679 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_59662. 
2024-12-12T02:08:01,679 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_61756. 
2024-12-12T02:08:01,680 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_63909. 
2024-12-12T02:08:01,680 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_66062. 
2024-12-12T02:08:01,680 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_68223. 
2024-12-12T02:08:01,680 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_70448. 
2024-12-12T02:08:01,681 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_72609. 
2024-12-12T02:08:01,681 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_74738. 
2024-12-12T02:08:01,681 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_76969. 
2024-12-12T02:08:01,681 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_79076. 
2024-12-12T02:08:01,681 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_81194. 
2024-12-12T02:08:01,682 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_83372. 
2024-12-12T02:08:01,682 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_85457. 
2024-12-12T02:08:01,682 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_87547. 
2024-12-12T02:08:01,682 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_89721. 
2024-12-12T02:08:01,682 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_91865. 
2024-12-12T02:08:01,683 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_94090. 
2024-12-12T02:08:01,683 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_96220. 
2024-12-12T02:08:01,683 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_98307. 
2024-12-12T02:08:01,683 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_100437. 
2024-12-12T02:08:01,683 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_102535. 
2024-12-12T02:08:01,683 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_104622. 
2024-12-12T02:08:01,684 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_106741. 
2024-12-12T02:08:01,684 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_109008. 
2024-12-12T02:08:01,684 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_111221. 
2024-12-12T02:08:01,684 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_113399. 
2024-12-12T02:08:01,685 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_115487. 
2024-12-12T02:08:01,685 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_117679. 
2024-12-12T02:08:01,685 DEBUG [hfile-prefetch-1733969275320 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 0379d42162114cd387240baaf64574a0_117964. 
2024-12-12T02:08:01,685 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch0/0379d42162114cd387240baaf64574a0, with total blocks 57, and data blocks 56
2024-12-12T02:08:01,685 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map
2024-12-12T02:08:01,685 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch0/0379d42162114cd387240baaf64574a0, but only 0 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue.
2024-12-12T02:08:01,685 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch0/0379d42162114cd387240baaf64574a0. Will sleep 100ms and try the verification again.
2024-12-12T02:08:01,786 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch0/0379d42162114cd387240baaf64574a0, with total blocks 57, and data blocks 56
2024-12-12T02:08:01,786 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2285): iterating over 2 entries in the backing map
2024-12-12T02:08:01,786 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 0
2024-12-12T02:08:01,787 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 2119
2024-12-12T02:08:01,787 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch0/0379d42162114cd387240baaf64574a0, but only 2 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue.
2024-12-12T02:08:01,787 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch0/0379d42162114cd387240baaf64574a0. Will sleep 100ms and try the verification again.
2024-12-12T02:08:01,887 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch0/0379d42162114cd387240baaf64574a0, with total blocks 57, and data blocks 56
2024-12-12T02:08:01,888 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2285): iterating over 57 entries in the backing map
2024-12-12T02:08:01,888 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 0
2024-12-12T02:08:01,888 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 2119
2024-12-12T02:08:01,888 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 4243
2024-12-12T02:08:01,888 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 6384
2024-12-12T02:08:01,888 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 8508
2024-12-12T02:08:01,888 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 10599
2024-12-12T02:08:01,888 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 12686
2024-12-12T02:08:01,888 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 14833
2024-12-12T02:08:01,888 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 16922
2024-12-12T02:08:01,889 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 19036
2024-12-12T02:08:01,889 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 21230
2024-12-12T02:08:01,889 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 23332
2024-12-12T02:08:01,889 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 25536
2024-12-12T02:08:01,889 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 27660
2024-12-12T02:08:01,889 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 29749
2024-12-12T02:08:01,889 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 31866
2024-12-12T02:08:01,889 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 33968
2024-12-12T02:08:01,889 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 36124
2024-12-12T02:08:01,889 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 38301
2024-12-12T02:08:01,889 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 40437
2024-12-12T02:08:01,890 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 42541
2024-12-12T02:08:01,890 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 44643
2024-12-12T02:08:01,890 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 46814
2024-12-12T02:08:01,890 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 48969
2024-12-12T02:08:01,890 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 51073
2024-12-12T02:08:01,890 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 53231
2024-12-12T02:08:01,890 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 55363
2024-12-12T02:08:01,890 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 57498
2024-12-12T02:08:01,890 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 59662
2024-12-12T02:08:01,890 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 61756
2024-12-12T02:08:01,891 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 63909
2024-12-12T02:08:01,891 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 66062
2024-12-12T02:08:01,891 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 68223
2024-12-12T02:08:01,891 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 70448
2024-12-12T02:08:01,891 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 72609
2024-12-12T02:08:01,891 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 74738
2024-12-12T02:08:01,891 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 76969
2024-12-12T02:08:01,891 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 79076
2024-12-12T02:08:01,891 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 81194
2024-12-12T02:08:01,891 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 83372
2024-12-12T02:08:01,891 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 85457
2024-12-12T02:08:01,892 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 87547
2024-12-12T02:08:01,892 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 89721
2024-12-12T02:08:01,892 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 91865
2024-12-12T02:08:01,892 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 94090
2024-12-12T02:08:01,892 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 96220
2024-12-12T02:08:01,892 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 98307
2024-12-12T02:08:01,892 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 100437
2024-12-12T02:08:01,892 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 102535
2024-12-12T02:08:01,892 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 104622
2024-12-12T02:08:01,892 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 106741
2024-12-12T02:08:01,892 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 109008
2024-12-12T02:08:01,892 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 111221
2024-12-12T02:08:01,893 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 113399
2024-12-12T02:08:01,893 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 115487
2024-12-12T02:08:01,893 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 117679
2024-12-12T02:08:01,893 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2293): found block for file 0379d42162114cd387240baaf64574a0 in the backing map. Acquiring read lock for offset 117964
2024-12-12T02:08:01,893 DEBUG [hfile-prefetch-1733969275320 {}] bucket.BucketCache(2305): File /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch0/0379d42162114cd387240baaf64574a0 has now been fully cached.
2024-12-12T02:08:01,894 DEBUG [hfile-prefetch-1733969275320 {}] hfile.PrefetchExecutor(121): Prefetch completed for 0379d42162114cd387240baaf64574a0
2024-12-12T02:08:02,745 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch1/8ed59a40e9a641a2ba1239db0aabde0a, delay=956 ms
2024-12-12T02:08:03,705 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_0. 
2024-12-12T02:08:03,707 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_2115. 
2024-12-12T02:08:03,708 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_4233. 
2024-12-12T02:08:03,709 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_6323. 
2024-12-12T02:08:03,709 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_8507. 
2024-12-12T02:08:03,711 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_10607. 
2024-12-12T02:08:03,711 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk
2024-12-12T02:08:03,711 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file
2024-12-12T02:08:03,711 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format.
2024-12-12T02:08:03,711 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 57, fullycachedFiles size: 1, chunkSize: 10000
2024-12-12T02:08:03,711 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_12701. 
2024-12-12T02:08:03,712 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_14823. 
2024-12-12T02:08:03,712 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_16966. 
2024-12-12T02:08:03,713 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_19101. 
2024-12-12T02:08:03,713 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_21236. 
2024-12-12T02:08:03,713 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_23476. 
2024-12-12T02:08:03,714 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_25625. 
2024-12-12T02:08:03,714 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_27791. 
2024-12-12T02:08:03,715 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_29984. 
2024-12-12T02:08:03,715 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_32149. 
2024-12-12T02:08:03,715 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_34316. 
2024-12-12T02:08:03,715 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_36428. 
2024-12-12T02:08:03,716 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_38537. 
2024-12-12T02:08:03,716 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_40662. 
2024-12-12T02:08:03,716 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_42780. 
2024-12-12T02:08:03,716 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 57, fullycachedFiles size: 1
2024-12-12T02:08:03,716 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 57, fullycachedFiles size: 1, file name: bucket.persistence1733969283711
2024-12-12T02:08:03,716 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming
2024-12-12T02:08:03,716 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_44965. 
2024-12-12T02:08:03,717 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_47050. 
2024-12-12T02:08:03,717 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_49174. 
2024-12-12T02:08:03,717 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_51389. 
2024-12-12T02:08:03,717 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_53623. 
2024-12-12T02:08:03,717 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_55709. 
2024-12-12T02:08:03,717 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_57952. 
2024-12-12T02:08:03,718 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_60091. 
2024-12-12T02:08:03,718 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_62194. 
2024-12-12T02:08:03,718 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_64408. 
2024-12-12T02:08:03,718 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_66502. 
2024-12-12T02:08:03,718 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_68665. 
2024-12-12T02:08:03,719 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_70788. 
2024-12-12T02:08:03,719 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_72950. 
2024-12-12T02:08:03,719 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_75148. 
2024-12-12T02:08:03,719 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_77309. 
2024-12-12T02:08:03,719 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_79519. 
2024-12-12T02:08:03,720 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_81726. 
2024-12-12T02:08:03,720 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_83874. 
2024-12-12T02:08:03,720 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_85975. 
2024-12-12T02:08:03,720 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_88088. 
2024-12-12T02:08:03,720 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_90216. 
2024-12-12T02:08:03,720 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_92342. 
2024-12-12T02:08:03,721 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_94481. 
2024-12-12T02:08:03,721 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_96652. 
2024-12-12T02:08:03,721 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_98773. 
2024-12-12T02:08:03,721 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_100924. 
2024-12-12T02:08:03,721 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_103077. 
2024-12-12T02:08:03,721 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_105206. 
2024-12-12T02:08:03,722 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_107385. 
2024-12-12T02:08:03,722 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_109581. 
2024-12-12T02:08:03,722 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_111731. 
2024-12-12T02:08:03,722 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_113934. 
2024-12-12T02:08:03,722 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_116039. 
2024-12-12T02:08:03,722 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_118222. 
2024-12-12T02:08:03,723 DEBUG [hfile-prefetch-1733969278650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8ed59a40e9a641a2ba1239db0aabde0a_118761. 
2024-12-12T02:08:03,723 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch1/8ed59a40e9a641a2ba1239db0aabde0a, with total blocks 57, and data blocks 56
2024-12-12T02:08:03,723 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2285): iterating over 57 entries in the backing map
2024-12-12T02:08:03,723 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch1/8ed59a40e9a641a2ba1239db0aabde0a, but only 0 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue.
2024-12-12T02:08:03,723 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch1/8ed59a40e9a641a2ba1239db0aabde0a. Will sleep 100ms and try the verification again.
2024-12-12T02:08:03,823 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch1/8ed59a40e9a641a2ba1239db0aabde0a, with total blocks 57, and data blocks 56
2024-12-12T02:08:03,824 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2285): iterating over 59 entries in the backing map
2024-12-12T02:08:03,824 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 0
2024-12-12T02:08:03,824 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 6323
2024-12-12T02:08:03,824 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch1/8ed59a40e9a641a2ba1239db0aabde0a, but only 2 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue.
2024-12-12T02:08:03,824 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch1/8ed59a40e9a641a2ba1239db0aabde0a. Will sleep 100ms and try the verification again.
2024-12-12T02:08:03,924 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch1/8ed59a40e9a641a2ba1239db0aabde0a, with total blocks 57, and data blocks 56
2024-12-12T02:08:03,925 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2285): iterating over 114 entries in the backing map
2024-12-12T02:08:03,925 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 0
2024-12-12T02:08:03,925 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 2115
2024-12-12T02:08:03,925 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 4233
2024-12-12T02:08:03,925 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 6323
2024-12-12T02:08:03,925 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 8507
2024-12-12T02:08:03,925 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 10607
2024-12-12T02:08:03,925 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 12701
2024-12-12T02:08:03,925 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 14823
2024-12-12T02:08:03,925 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 16966
2024-12-12T02:08:03,926 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 19101
2024-12-12T02:08:03,926 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 21236
2024-12-12T02:08:03,926 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 23476
2024-12-12T02:08:03,926 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 25625
2024-12-12T02:08:03,926 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 27791
2024-12-12T02:08:03,926 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 29984
2024-12-12T02:08:03,926 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 32149
2024-12-12T02:08:03,926 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 34316
2024-12-12T02:08:03,926 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 36428
2024-12-12T02:08:03,926 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 38537
2024-12-12T02:08:03,926 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 40662
2024-12-12T02:08:03,926 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 42780
2024-12-12T02:08:03,927 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 44965
2024-12-12T02:08:03,927 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 47050
2024-12-12T02:08:03,927 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 49174
2024-12-12T02:08:03,927 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 51389
2024-12-12T02:08:03,927 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 53623
2024-12-12T02:08:03,927 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 55709
2024-12-12T02:08:03,927 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 57952
2024-12-12T02:08:03,927 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 60091
2024-12-12T02:08:03,927 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 62194
2024-12-12T02:08:03,927 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 64408
2024-12-12T02:08:03,927 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 66502
2024-12-12T02:08:03,927 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 68665
2024-12-12T02:08:03,927 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 70788
2024-12-12T02:08:03,928 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 72950
2024-12-12T02:08:03,928 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 75148
2024-12-12T02:08:03,928 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 77309
2024-12-12T02:08:03,928 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 79519
2024-12-12T02:08:03,928 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 81726
2024-12-12T02:08:03,928 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 83874
2024-12-12T02:08:03,928 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 85975
2024-12-12T02:08:03,928 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 88088
2024-12-12T02:08:03,928 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 90216
2024-12-12T02:08:03,928 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 92342
2024-12-12T02:08:03,928 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 94481
2024-12-12T02:08:03,928 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 96652
2024-12-12T02:08:03,928 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 98773
2024-12-12T02:08:03,928 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 100924
2024-12-12T02:08:03,929 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 103077
2024-12-12T02:08:03,929 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 105206
2024-12-12T02:08:03,929 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 107385
2024-12-12T02:08:03,929 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 109581
2024-12-12T02:08:03,929 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 111731
2024-12-12T02:08:03,929 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 113934
2024-12-12T02:08:03,929 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 116039
2024-12-12T02:08:03,929 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 118222
2024-12-12T02:08:03,929 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2293): found block for file 8ed59a40e9a641a2ba1239db0aabde0a in the backing map. Acquiring read lock for offset 118761
2024-12-12T02:08:03,929 DEBUG [hfile-prefetch-1733969278650 {}] bucket.BucketCache(2305): File /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/TestPrefetch1/8ed59a40e9a641a2ba1239db0aabde0a has now been fully cached.
2024-12-12T02:08:03,930 DEBUG [hfile-prefetch-1733969278650 {}] hfile.PrefetchExecutor(121): Prefetch completed for 8ed59a40e9a641a2ba1239db0aabde0a
2024-12-12T02:08:06,717 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk
2024-12-12T02:08:06,717 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file
2024-12-12T02:08:06,718 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format.
2024-12-12T02:08:06,718 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 114, fullycachedFiles size: 2, chunkSize: 10000
2024-12-12T02:08:06,727 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 114, fullycachedFiles size: 2
2024-12-12T02:08:06,727 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 114, fullycachedFiles size: 2, file name: bucket.persistence1733969286718
2024-12-12T02:08:06,727 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming
2024-12-12T02:08:07,750 INFO  [Time-limited test {}] bucket.BucketCache(1738): Disabling cache
2024-12-12T02:08:07,751 INFO  [Time-limited test {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38cf2fb7-8fb6-6119-ce82-dac51eb8fd81/bucket.persistence
2024-12-12T02:08:07,751 INFO  [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false
2024-12-12T02:08:07,751 INFO  [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false
2024-12-12T02:08:07,751 INFO  [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false
2024-12-12T02:08:07,752 INFO  [Time-limited test {}] bucket.BucketCache(1768): Shutting down cache persister thread.
2024-12-12T02:08:07,752 WARN  [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread.
java.lang.InterruptedException: sleep interrupted
	at java.lang.Thread.sleep(Native Method) ~[?:?]
	at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?]
2024-12-12T02:08:07,762 DEBUG [Time-limited test {}] bucket.BucketCache(1487): Thread Time-limited test started persisting bucket cache to file
2024-12-12T02:08:07,762 DEBUG [Time-limited test {}] bucket.BucketCache(1494): Persist in new chunked persistence format.
2024-12-12T02:08:07,762 DEBUG [Time-limited test {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 114, fullycachedFiles size: 2, chunkSize: 10000
2024-12-12T02:08:07,769 DEBUG [Time-limited test {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 114, fullycachedFiles size: 2
2024-12-12T02:08:07,770 DEBUG [Time-limited test {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 114, fullycachedFiles size: 2, file name: bucket.persistence1733969287762
2024-12-12T02:08:07,770 DEBUG [Time-limited test {}] bucket.BucketCache(1509): Thread Time-limited test finished persisting bucket cache to file, renaming
2024-12-12T02:08:07,770 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-12T02:08:07,772 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-12T02:08:07,777 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestBucketCachePersister#testPrefetchPersistenceCrash Thread=23 (was 22)
Potentially hanging thread: hfile-prefetch-1733969280740
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=292 (was 290) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=718 (was 772), ProcessCount=11 (was 11), AvailableMemoryMB=8947 (was 8955)