2024-12-10 12:05:53,132 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@57bc27f5
2024-12-10 12:05:53,149 main DEBUG Took 0.014160 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging
2024-12-10 12:05:53,149 main DEBUG PluginManager 'Core' found 129 plugins
2024-12-10 12:05:53,150 main DEBUG PluginManager 'Level' found 0 plugins
2024-12-10 12:05:53,151 main DEBUG PluginManager 'Lookup' found 16 plugins
2024-12-10 12:05:53,153 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-10 12:05:53,163 main DEBUG PluginManager 'TypeConverter' found 26 plugins
2024-12-10 12:05:53,180 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-10 12:05:53,182 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-10 12:05:53,183 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-10 12:05:53,183 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-10 12:05:53,184 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-10 12:05:53,184 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-10 12:05:53,185 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-10 12:05:53,185 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-10 12:05:53,186 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-10 12:05:53,186 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-10 12:05:53,188 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-10 12:05:53,188 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-10 12:05:53,189 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-10 12:05:53,189 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-10 12:05:53,190 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-10 12:05:53,190 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-10 12:05:53,191 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-10 12:05:53,191 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-10 12:05:53,192 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-10 12:05:53,192 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-10 12:05:53,192 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-10 12:05:53,193 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-10 12:05:53,193 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-10 12:05:53,194 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-10 12:05:53,194 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-10 12:05:53,195 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger].
2024-12-10 12:05:53,197 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-10 12:05:53,198 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin].
2024-12-10 12:05:53,201 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root})
2024-12-10 12:05:53,201 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout].
2024-12-10 12:05:53,203 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null")
2024-12-10 12:05:53,204 main DEBUG PluginManager 'Converter' found 47 plugins
2024-12-10 12:05:53,214 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender].
2024-12-10 12:05:53,217 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={})
2024-12-10 12:05:53,220 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR
2024-12-10 12:05:53,220 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin].
2024-12-10 12:05:53,221 main DEBUG createAppenders(={Console})
2024-12-10 12:05:53,222 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@57bc27f5 initialized
2024-12-10 12:05:53,222 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@57bc27f5
2024-12-10 12:05:53,223 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@57bc27f5 OK.
2024-12-10 12:05:53,223 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1
2024-12-10 12:05:53,224 main DEBUG OutputStream closed
2024-12-10 12:05:53,224 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true
2024-12-10 12:05:53,224 main DEBUG Appender DefaultConsole-1 stopped with status true
2024-12-10 12:05:53,225 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5a56cdac OK
2024-12-10 12:05:53,316 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6
2024-12-10 12:05:53,319 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger
2024-12-10 12:05:53,320 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector
2024-12-10 12:05:53,320 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=
2024-12-10 12:05:53,321 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory
2024-12-10 12:05:53,321 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter
2024-12-10 12:05:53,322 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper
2024-12-10 12:05:53,322 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j
2024-12-10 12:05:53,322 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl
2024-12-10 12:05:53,323 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans
2024-12-10 12:05:53,323 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase
2024-12-10 12:05:53,324 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop
2024-12-10 12:05:53,324 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers
2024-12-10 12:05:53,324 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices
2024-12-10 12:05:53,325 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig
2024-12-10 12:05:53,325 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel
2024-12-10 12:05:53,325 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore
2024-12-10 12:05:53,326 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console
2024-12-10 12:05:53,328 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-12-10 12:05:53,329 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@1608bcbd) with optional ClassLoader: null
2024-12-10 12:05:53,329 main DEBUG Shutdown hook enabled. Registering a new one.
2024-12-10 12:05:53,330 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@1608bcbd] started OK.
2024-12-10T12:05:53,344 INFO  [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.bucket.TestBucketCachePersister timeout: 13 mins
2024-12-10 12:05:53,347 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED)
2024-12-10 12:05:53,348 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-12-10T12:05:53,564 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca
2024-12-10T12:05:53,589 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestBucketCachePersister#testPrefetchBlockEvictionWhilePrefetchRunning Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=165, ProcessCount=11, AvailableMemoryMB=6854
2024-12-10T12:05:53,671 WARN  [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
2024-12-10T12:05:53,754 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-10T12:05:53,776 INFO  [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/testPrefetchBlockEvictionWhilePrefetchRunning
2024-12-10T12:05:53,785 INFO  [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false
2024-12-10T12:05:53,787 INFO  [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread
2024-12-10T12:05:53,790 INFO  [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 200
2024-12-10T12:05:53,791 INFO  [Thread-5 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file
2024-12-10T12:05:53,792 WARN  [Thread-5 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache.
2024-12-10T12:05:53,797 INFO  [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/testPrefetchBlockEvictionWhilePrefetchRunning, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator
2024-12-10T12:05:53,799 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-10T12:05:53,800 INFO  [Thread-5 {}] bucket.BucketAllocator(343): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size))
2024-12-10T12:05:53,800 INFO  [Thread-5 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/bucket.persistence is complete.
2024-12-10T12:05:53,833 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout
2024-12-10T12:05:53,890 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a4979ce1792047e5b27d8bb5c620e131
2024-12-10T12:05:53,974 WARN  [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-12-10T12:05:54,006 INFO  [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl
2024-12-10T12:05:54,037 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is 271, key is aaaaaaaaaaaaaaaaaaaaaaaabbbababb/zBoDU00Z9:6/7175731145079643019/Put/seqid=0
2024-12-10T12:05:54,123 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131, delay=968 ms
2024-12-10T12:05:54,157 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,167 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,178 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,188 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,199 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,209 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,220 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,230 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,241 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,251 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,262 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,272 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,283 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,293 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,304 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,314 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,325 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,335 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,345 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,356 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,366 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,377 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,387 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,398 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,408 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,418 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,429 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,439 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,450 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,460 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,471 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,481 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,491 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,501 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,512 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,522 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,532 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,543 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,553 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,564 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,574 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,585 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,595 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,606 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,616 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,627 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,638 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,648 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,659 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,670 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,680 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,690 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,701 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,711 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,722 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,732 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,742 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,752 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,763 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,773 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,783 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,794 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,804 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,814 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,825 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,835 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,845 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,856 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,866 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,876 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,887 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,897 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,907 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,918 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,928 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,939 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,949 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,960 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,971 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,981 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:54,992 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:55,003 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:55,013 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:55,024 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:55,034 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:55,045 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:55,055 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:55,066 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:55,076 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:55,087 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:55,097 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:55,104 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_0. 
2024-12-10T12:05:55,108 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:55,118 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:55,128 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:55,138 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_2104. 
2024-12-10T12:05:55,138 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_4298. 
2024-12-10T12:05:55,139 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:55,139 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_6459. 
2024-12-10T12:05:55,140 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_8607. 
2024-12-10T12:05:55,140 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_10698. 
2024-12-10T12:05:55,141 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_12866. 
2024-12-10T12:05:55,141 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_15096. 
2024-12-10T12:05:55,141 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_17220. 
2024-12-10T12:05:55,142 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_19412. 
2024-12-10T12:05:55,142 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_21565. 
2024-12-10T12:05:55,143 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_23681. 
2024-12-10T12:05:55,143 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_25829. 
2024-12-10T12:05:55,144 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_28009. 
2024-12-10T12:05:55,144 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_30135. 
2024-12-10T12:05:55,145 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_32281. 
2024-12-10T12:05:55,145 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_34425. 
2024-12-10T12:05:55,146 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_36545. 
2024-12-10T12:05:55,146 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_38703. 
2024-12-10T12:05:55,147 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_40880. 
2024-12-10T12:05:55,147 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_43038. 
2024-12-10T12:05:55,148 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_45209. 
2024-12-10T12:05:55,148 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_47297. 
2024-12-10T12:05:55,149 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_49395. 
2024-12-10T12:05:55,149 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:55,149 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_51507. 
2024-12-10T12:05:55,150 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_53719. 
2024-12-10T12:05:55,150 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_55811. 
2024-12-10T12:05:55,151 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_58022. 
2024-12-10T12:05:55,151 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_60220. 
2024-12-10T12:05:55,152 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_62348. 
2024-12-10T12:05:55,152 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_64522. 
2024-12-10T12:05:55,153 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_66693. 
2024-12-10T12:05:55,153 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_68828. 
2024-12-10T12:05:55,153 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_70980. 
2024-12-10T12:05:55,154 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_73166. 
2024-12-10T12:05:55,154 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_75286. 
2024-12-10T12:05:55,155 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_77532. 
2024-12-10T12:05:55,155 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_79669. 
2024-12-10T12:05:55,156 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_81863. 
2024-12-10T12:05:55,156 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_83960. 
2024-12-10T12:05:55,157 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_86087. 
2024-12-10T12:05:55,157 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_88228. 
2024-12-10T12:05:55,158 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_90333. 
2024-12-10T12:05:55,158 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_92441. 
2024-12-10T12:05:55,159 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_94546. 
2024-12-10T12:05:55,159 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:55,159 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_96717. 
2024-12-10T12:05:55,160 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_98831. 
2024-12-10T12:05:55,160 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_100941. 
2024-12-10T12:05:55,161 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_103070. 
2024-12-10T12:05:55,161 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_105209. 
2024-12-10T12:05:55,162 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_107366. 
2024-12-10T12:05:55,162 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_109556. 
2024-12-10T12:05:55,163 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_111699. 
2024-12-10T12:05:55,163 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_113894. 
2024-12-10T12:05:55,163 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_116029. 
2024-12-10T12:05:55,164 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_118125. 
2024-12-10T12:05:55,164 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a4979ce1792047e5b27d8bb5c620e131_119429. 
2024-12-10T12:05:55,165 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131, with total blocks 57, and data blocks 56
2024-12-10T12:05:55,165 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map
2024-12-10T12:05:55,166 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131, but only 0 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue.
2024-12-10T12:05:55,167 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131. Will sleep 100ms and try the verification again.
2024-12-10T12:05:55,169 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:55,180 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:55,190 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:55,200 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:55,211 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:55,212 DEBUG [Time-limited test {}] bucket.BucketCache(856): removed key a4979ce1792047e5b27d8bb5c620e131_0 from back map with offset lock 525312 in the evict process
2024-12-10T12:05:55,213 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(182): Attempted eviction for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131. Succeeded? true
2024-12-10T12:05:55,223 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:55,233 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:55,243 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:55,254 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:55,264 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:55,267 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131, with total blocks 57, and data blocks 56
2024-12-10T12:05:55,267 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2285): iterating over 56 entries in the backing map
2024-12-10T12:05:55,267 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 2104
2024-12-10T12:05:55,268 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 4298
2024-12-10T12:05:55,268 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 6459
2024-12-10T12:05:55,268 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 8607
2024-12-10T12:05:55,268 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 10698
2024-12-10T12:05:55,268 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 12866
2024-12-10T12:05:55,269 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 15096
2024-12-10T12:05:55,269 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 17220
2024-12-10T12:05:55,269 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 19412
2024-12-10T12:05:55,269 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 21565
2024-12-10T12:05:55,269 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 23681
2024-12-10T12:05:55,269 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 25829
2024-12-10T12:05:55,270 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 28009
2024-12-10T12:05:55,270 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 30135
2024-12-10T12:05:55,270 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 32281
2024-12-10T12:05:55,270 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 34425
2024-12-10T12:05:55,270 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 36545
2024-12-10T12:05:55,270 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 38703
2024-12-10T12:05:55,270 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 40880
2024-12-10T12:05:55,270 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 43038
2024-12-10T12:05:55,271 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 45209
2024-12-10T12:05:55,271 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 47297
2024-12-10T12:05:55,271 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 49395
2024-12-10T12:05:55,271 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 51507
2024-12-10T12:05:55,271 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 53719
2024-12-10T12:05:55,271 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 55811
2024-12-10T12:05:55,272 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 58022
2024-12-10T12:05:55,272 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 60220
2024-12-10T12:05:55,272 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 62348
2024-12-10T12:05:55,272 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 64522
2024-12-10T12:05:55,272 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 66693
2024-12-10T12:05:55,272 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 68828
2024-12-10T12:05:55,272 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 70980
2024-12-10T12:05:55,273 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 73166
2024-12-10T12:05:55,273 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 75286
2024-12-10T12:05:55,273 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 77532
2024-12-10T12:05:55,273 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 79669
2024-12-10T12:05:55,273 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 81863
2024-12-10T12:05:55,273 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 83960
2024-12-10T12:05:55,274 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 86087
2024-12-10T12:05:55,274 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 88228
2024-12-10T12:05:55,274 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 90333
2024-12-10T12:05:55,274 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 92441
2024-12-10T12:05:55,274 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 94546
2024-12-10T12:05:55,274 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 96717
2024-12-10T12:05:55,274 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 98831
2024-12-10T12:05:55,275 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 100941
2024-12-10T12:05:55,275 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131 is still running.
2024-12-10T12:05:55,275 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 103070
2024-12-10T12:05:55,275 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 105209
2024-12-10T12:05:55,275 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 107366
2024-12-10T12:05:55,275 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 109556
2024-12-10T12:05:55,275 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 111699
2024-12-10T12:05:55,275 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 113894
2024-12-10T12:05:55,275 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 116029
2024-12-10T12:05:55,276 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 118125
2024-12-10T12:05:55,276 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file a4979ce1792047e5b27d8bb5c620e131 in the backing map. Acquiring read lock for offset 119429
2024-12-10T12:05:55,276 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131, but only 55 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue.
2024-12-10T12:05:55,276 INFO  [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2331): We found only 55 data blocks cached from a total of 56 for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/a4979ce1792047e5b27d8bb5c620e131, but no blocks pending caching. Maybe cache is full or evictions happened concurrently to cache prefetch.
2024-12-10T12:05:55,277 DEBUG [hfile-prefetch-1733832354125 {}] hfile.PrefetchExecutor(121): Prefetch completed for a4979ce1792047e5b27d8bb5c620e131
2024-12-10T12:05:55,285 INFO  [Time-limited test {}] bucket.BucketCache(1738): Disabling cache
2024-12-10T12:05:55,286 INFO  [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false
2024-12-10T12:05:55,286 INFO  [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false
2024-12-10T12:05:55,286 INFO  [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false
2024-12-10T12:05:55,287 INFO  [Time-limited test {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/bucket.persistence
2024-12-10T12:05:55,288 INFO  [Time-limited test {}] bucket.BucketCache(1768): Shutting down cache persister thread.
2024-12-10T12:05:55,288 WARN  [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread.
java.lang.InterruptedException: sleep interrupted
	at java.lang.Thread.sleep(Native Method) ~[?:?]
	at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?]
2024-12-10T12:05:55,298 DEBUG [Time-limited test {}] bucket.BucketCache(1487): Thread Time-limited test started persisting bucket cache to file
2024-12-10T12:05:55,299 DEBUG [Time-limited test {}] bucket.BucketCache(1494): Persist in new chunked persistence format.
2024-12-10T12:05:55,299 DEBUG [Time-limited test {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 56, fullycachedFiles size: 0, chunkSize: 10000
2024-12-10T12:05:55,383 DEBUG [Time-limited test {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 56, fullycachedFiles size: 0
2024-12-10T12:05:55,383 DEBUG [Time-limited test {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 56, fullycachedFiles size: 0, file name: bucket.persistence1733832355298
2024-12-10T12:05:55,383 DEBUG [Time-limited test {}] bucket.BucketCache(1509): Thread Time-limited test finished persisting bucket cache to file, renaming
2024-12-10T12:05:55,384 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-10T12:05:55,384 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-10T12:05:55,387 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-10T12:05:55,392 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestBucketCachePersister#testPrefetchBlockEvictionWhilePrefetchRunning Thread=20 (was 12)
Potentially hanging thread: Timer for 'HBase' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: ForkJoinPool.commonPool-worker-2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: HBase-Metrics2-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool.commonPool-worker-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: weak-ref-cleaner-strictcontextstorage
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool.commonPool-worker-3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: hfile-prefetch-1733832354125
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=288 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=192 (was 165) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6803 (was 6854)
2024-12-10T12:05:55,397 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestBucketCachePersister#testPrefetchPersistenceCrashNegative Thread=20, OpenFileDescriptor=288, MaxFileDescriptor=1048576, SystemLoadAverage=192, ProcessCount=11, AvailableMemoryMB=6803
2024-12-10T12:05:55,398 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-10T12:05:55,400 INFO  [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/testPrefetchPersistenceCrashNegative
2024-12-10T12:05:55,401 INFO  [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false
2024-12-10T12:05:55,401 INFO  [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread
2024-12-10T12:05:55,401 INFO  [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 9223372036854775807
2024-12-10T12:05:55,403 INFO  [Thread-12 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file
2024-12-10T12:05:55,403 INFO  [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/testPrefetchPersistenceCrashNegative, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator
2024-12-10T12:05:55,403 WARN  [Thread-12 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache.
2024-12-10T12:05:55,403 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-10T12:05:55,404 INFO  [Thread-12 {}] bucket.BucketAllocator(343): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size))
2024-12-10T12:05:55,404 INFO  [Thread-12 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/bucket.persistence is complete.
2024-12-10T12:05:55,410 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 307af276319d4fdab47ae14f8da8515b
2024-12-10T12:05:55,425 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch2/307af276319d4fdab47ae14f8da8515b is 239, key is aaaaaaaaaaaaaaaaaaaaaabaaaabbbbb/SJS1rFWEyNzOag1J9/2508873133482725456/DeleteFamilyVersion/seqid=0
2024-12-10T12:05:55,428 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch2/307af276319d4fdab47ae14f8da8515b, delay=976 ms
2024-12-10T12:05:56,406 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_0. 
2024-12-10T12:05:56,408 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_2153. 
2024-12-10T12:05:56,409 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_4328. 
2024-12-10T12:05:56,410 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_6449. 
2024-12-10T12:05:56,411 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_8565. 
2024-12-10T12:05:56,412 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_10708. 
2024-12-10T12:05:56,413 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_12838. 
2024-12-10T12:05:56,414 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_14958. 
2024-12-10T12:05:56,415 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_17161. 
2024-12-10T12:05:56,415 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_19296. 
2024-12-10T12:05:56,415 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_21544. 
2024-12-10T12:05:56,416 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_23727. 
2024-12-10T12:05:56,416 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_25813. 
2024-12-10T12:05:56,416 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_27956. 
2024-12-10T12:05:56,416 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_30109. 
2024-12-10T12:05:56,417 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_32291. 
2024-12-10T12:05:56,417 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_34436. 
2024-12-10T12:05:56,417 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_36559. 
2024-12-10T12:05:56,418 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_38648. 
2024-12-10T12:05:56,418 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_40799. 
2024-12-10T12:05:56,418 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_42924. 
2024-12-10T12:05:56,418 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_45030. 
2024-12-10T12:05:56,419 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_47175. 
2024-12-10T12:05:56,419 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_49309. 
2024-12-10T12:05:56,419 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_51524. 
2024-12-10T12:05:56,419 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_53654. 
2024-12-10T12:05:56,420 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_55750. 
2024-12-10T12:05:56,420 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_57991. 
2024-12-10T12:05:56,420 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_60080. 
2024-12-10T12:05:56,421 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_62182. 
2024-12-10T12:05:56,421 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_64365. 
2024-12-10T12:05:56,421 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_66496. 
2024-12-10T12:05:56,421 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_68617. 
2024-12-10T12:05:56,422 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_70748. 
2024-12-10T12:05:56,422 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_72952. 
2024-12-10T12:05:56,422 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_75145. 
2024-12-10T12:05:56,422 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_77243. 
2024-12-10T12:05:56,423 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_79361. 
2024-12-10T12:05:56,423 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_81508. 
2024-12-10T12:05:56,423 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_83624. 
2024-12-10T12:05:56,423 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_85733. 
2024-12-10T12:05:56,424 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_87831. 
2024-12-10T12:05:56,424 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_89971. 
2024-12-10T12:05:56,424 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_92107. 
2024-12-10T12:05:56,424 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_94272. 
2024-12-10T12:05:56,425 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_96400. 
2024-12-10T12:05:56,425 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_98506. 
2024-12-10T12:05:56,425 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_100672. 
2024-12-10T12:05:56,425 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_102825. 
2024-12-10T12:05:56,425 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_104934. 
2024-12-10T12:05:56,426 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_107046. 
2024-12-10T12:05:56,426 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_109273. 
2024-12-10T12:05:56,426 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_111416. 
2024-12-10T12:05:56,426 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_113524. 
2024-12-10T12:05:56,427 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_115617. 
2024-12-10T12:05:56,427 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_117766. 
2024-12-10T12:05:56,427 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 307af276319d4fdab47ae14f8da8515b_118067. 
2024-12-10T12:05:56,427 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch2/307af276319d4fdab47ae14f8da8515b, with total blocks 57, and data blocks 56
2024-12-10T12:05:56,428 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map
2024-12-10T12:05:56,428 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch2/307af276319d4fdab47ae14f8da8515b, but only 0 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue.
2024-12-10T12:05:56,428 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch2/307af276319d4fdab47ae14f8da8515b. Will sleep 100ms and try the verification again.
2024-12-10T12:05:56,528 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch2/307af276319d4fdab47ae14f8da8515b, with total blocks 57, and data blocks 56
2024-12-10T12:05:56,528 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2285): iterating over 3 entries in the backing map
2024-12-10T12:05:56,529 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 0
2024-12-10T12:05:56,529 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 2153
2024-12-10T12:05:56,529 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 10708
2024-12-10T12:05:56,530 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch2/307af276319d4fdab47ae14f8da8515b, but only 3 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue.
2024-12-10T12:05:56,530 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch2/307af276319d4fdab47ae14f8da8515b. Will sleep 100ms and try the verification again.
2024-12-10T12:05:56,631 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch2/307af276319d4fdab47ae14f8da8515b, with total blocks 57, and data blocks 56
2024-12-10T12:05:56,631 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2285): iterating over 57 entries in the backing map
2024-12-10T12:05:56,631 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 0
2024-12-10T12:05:56,632 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 2153
2024-12-10T12:05:56,632 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 4328
2024-12-10T12:05:56,632 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 6449
2024-12-10T12:05:56,632 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 8565
2024-12-10T12:05:56,633 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 10708
2024-12-10T12:05:56,633 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 12838
2024-12-10T12:05:56,633 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 14958
2024-12-10T12:05:56,633 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 17161
2024-12-10T12:05:56,634 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 19296
2024-12-10T12:05:56,634 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 21544
2024-12-10T12:05:56,634 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 23727
2024-12-10T12:05:56,634 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 25813
2024-12-10T12:05:56,635 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 27956
2024-12-10T12:05:56,635 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 30109
2024-12-10T12:05:56,635 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 32291
2024-12-10T12:05:56,636 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 34436
2024-12-10T12:05:56,636 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 36559
2024-12-10T12:05:56,636 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 38648
2024-12-10T12:05:56,636 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 40799
2024-12-10T12:05:56,636 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 42924
2024-12-10T12:05:56,637 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 45030
2024-12-10T12:05:56,637 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 47175
2024-12-10T12:05:56,637 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 49309
2024-12-10T12:05:56,638 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 51524
2024-12-10T12:05:56,638 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 53654
2024-12-10T12:05:56,638 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 55750
2024-12-10T12:05:56,638 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 57991
2024-12-10T12:05:56,639 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 60080
2024-12-10T12:05:56,639 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 62182
2024-12-10T12:05:56,639 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 64365
2024-12-10T12:05:56,639 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 66496
2024-12-10T12:05:56,639 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 68617
2024-12-10T12:05:56,639 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 70748
2024-12-10T12:05:56,639 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 72952
2024-12-10T12:05:56,639 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 75145
2024-12-10T12:05:56,639 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 77243
2024-12-10T12:05:56,639 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 79361
2024-12-10T12:05:56,639 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 81508
2024-12-10T12:05:56,640 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 83624
2024-12-10T12:05:56,640 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 85733
2024-12-10T12:05:56,640 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 87831
2024-12-10T12:05:56,640 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 89971
2024-12-10T12:05:56,640 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 92107
2024-12-10T12:05:56,640 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 94272
2024-12-10T12:05:56,640 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 96400
2024-12-10T12:05:56,640 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 98506
2024-12-10T12:05:56,640 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 100672
2024-12-10T12:05:56,640 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 102825
2024-12-10T12:05:56,640 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 104934
2024-12-10T12:05:56,640 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 107046
2024-12-10T12:05:56,640 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 109273
2024-12-10T12:05:56,640 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 111416
2024-12-10T12:05:56,641 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 113524
2024-12-10T12:05:56,641 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 115617
2024-12-10T12:05:56,641 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 117766
2024-12-10T12:05:56,641 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 307af276319d4fdab47ae14f8da8515b in the backing map. Acquiring read lock for offset 118067
2024-12-10T12:05:56,641 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2305): File /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch2/307af276319d4fdab47ae14f8da8515b has now been fully cached.
2024-12-10T12:05:56,641 DEBUG [hfile-prefetch-1733832354125 {}] hfile.PrefetchExecutor(121): Prefetch completed for 307af276319d4fdab47ae14f8da8515b
2024-12-10T12:05:57,431 INFO  [Time-limited test {}] bucket.BucketCache(1738): Disabling cache
2024-12-10T12:05:57,432 INFO  [Time-limited test {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/bucket.persistence
2024-12-10T12:05:57,432 INFO  [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false
2024-12-10T12:05:57,432 INFO  [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false
2024-12-10T12:05:57,432 INFO  [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false
2024-12-10T12:05:57,432 INFO  [Time-limited test {}] bucket.BucketCache(1768): Shutting down cache persister thread.
2024-12-10T12:05:57,432 WARN  [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread.
java.lang.InterruptedException: sleep interrupted
	at java.lang.Thread.sleep(Native Method) ~[?:?]
	at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?]
2024-12-10T12:05:57,442 DEBUG [Time-limited test {}] bucket.BucketCache(1487): Thread Time-limited test started persisting bucket cache to file
2024-12-10T12:05:57,443 DEBUG [Time-limited test {}] bucket.BucketCache(1494): Persist in new chunked persistence format.
2024-12-10T12:05:57,443 DEBUG [Time-limited test {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 57, fullycachedFiles size: 1, chunkSize: 10000
2024-12-10T12:05:57,447 DEBUG [Time-limited test {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 57, fullycachedFiles size: 1
2024-12-10T12:05:57,448 DEBUG [Time-limited test {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 57, fullycachedFiles size: 1, file name: bucket.persistence1733832357442
2024-12-10T12:05:57,448 DEBUG [Time-limited test {}] bucket.BucketCache(1509): Thread Time-limited test finished persisting bucket cache to file, renaming
2024-12-10T12:05:57,448 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-10T12:05:57,449 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-10T12:05:57,454 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestBucketCachePersister#testPrefetchPersistenceCrashNegative Thread=21 (was 20)
Potentially hanging thread: hfile-prefetch-1733832355428
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=289 (was 288) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=192 (was 192), ProcessCount=11 (was 11), AvailableMemoryMB=6798 (was 6803)
2024-12-10T12:05:57,458 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestBucketCachePersister#testPrefetchListUponBlockEviction Thread=21, OpenFileDescriptor=289, MaxFileDescriptor=1048576, SystemLoadAverage=192, ProcessCount=11, AvailableMemoryMB=6797
2024-12-10T12:05:57,459 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-10T12:05:57,461 INFO  [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/testPrefetchListUponBlockEviction
2024-12-10T12:05:57,462 INFO  [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false
2024-12-10T12:05:57,462 INFO  [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread
2024-12-10T12:05:57,462 INFO  [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 200
2024-12-10T12:05:57,462 INFO  [Thread-17 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file
2024-12-10T12:05:57,462 WARN  [Thread-17 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache.
2024-12-10T12:05:57,463 INFO  [Thread-17 {}] bucket.BucketAllocator(343): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size))
2024-12-10T12:05:57,463 INFO  [Thread-17 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/bucket.persistence is complete.
2024-12-10T12:05:57,468 INFO  [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/testPrefetchListUponBlockEviction, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator
2024-12-10T12:05:57,469 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-10T12:05:57,477 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 1c642949999f491caefb51a05c88376d
2024-12-10T12:05:57,486 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/1c642949999f491caefb51a05c88376d is 234, key is aaaaaaaaaaaaaaaaaaaaaabbabbbbabb/Ielvbchvz_b:w/7528219297980958774/Delete/seqid=0
2024-12-10T12:05:57,489 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/1c642949999f491caefb51a05c88376d, delay=907 ms
2024-12-10T12:05:57,662 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk
2024-12-10T12:05:57,662 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file
2024-12-10T12:05:57,663 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format.
2024-12-10T12:05:57,663 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000
2024-12-10T12:05:57,665 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0
2024-12-10T12:05:57,666 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733832357663
2024-12-10T12:05:57,666 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming
2024-12-10T12:05:58,397 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_0. 
2024-12-10T12:05:58,398 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_2129. 
2024-12-10T12:05:58,399 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_4287. 
2024-12-10T12:05:58,399 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_6457. 
2024-12-10T12:05:58,400 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_8587. 
2024-12-10T12:05:58,400 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_10676. 
2024-12-10T12:05:58,400 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_12803. 
2024-12-10T12:05:58,400 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_14998. 
2024-12-10T12:05:58,401 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_17161. 
2024-12-10T12:05:58,401 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_19314. 
2024-12-10T12:05:58,401 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_21494. 
2024-12-10T12:05:58,401 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_23643. 
2024-12-10T12:05:58,402 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_25751. 
2024-12-10T12:05:58,402 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_27935. 
2024-12-10T12:05:58,402 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_30028. 
2024-12-10T12:05:58,402 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_32130. 
2024-12-10T12:05:58,403 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_34273. 
2024-12-10T12:05:58,403 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_36365. 
2024-12-10T12:05:58,403 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_38496. 
2024-12-10T12:05:58,403 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_40585. 
2024-12-10T12:05:58,404 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_42690. 
2024-12-10T12:05:58,404 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_44940. 
2024-12-10T12:05:58,404 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_47114. 
2024-12-10T12:05:58,404 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_49267. 
2024-12-10T12:05:58,404 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_51381. 
2024-12-10T12:05:58,405 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_53494. 
2024-12-10T12:05:58,405 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_55625. 
2024-12-10T12:05:58,405 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_57840. 
2024-12-10T12:05:58,405 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_60019. 
2024-12-10T12:05:58,406 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_62120. 
2024-12-10T12:05:58,406 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_64302. 
2024-12-10T12:05:58,406 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_66469. 
2024-12-10T12:05:58,406 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_68645. 
2024-12-10T12:05:58,406 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_70820. 
2024-12-10T12:05:58,407 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_72922. 
2024-12-10T12:05:58,407 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_75052. 
2024-12-10T12:05:58,407 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_77156. 
2024-12-10T12:05:58,407 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_79268. 
2024-12-10T12:05:58,407 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_81388. 
2024-12-10T12:05:58,408 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_83526. 
2024-12-10T12:05:58,408 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_85701. 
2024-12-10T12:05:58,408 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_87843. 
2024-12-10T12:05:58,408 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_90078. 
2024-12-10T12:05:58,409 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_92229. 
2024-12-10T12:05:58,409 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_94392. 
2024-12-10T12:05:58,409 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_96562. 
2024-12-10T12:05:58,409 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_98731. 
2024-12-10T12:05:58,409 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_100855. 
2024-12-10T12:05:58,410 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_102955. 
2024-12-10T12:05:58,410 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_105042. 
2024-12-10T12:05:58,410 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_107264. 
2024-12-10T12:05:58,410 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_109410. 
2024-12-10T12:05:58,411 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_111576. 
2024-12-10T12:05:58,411 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_113729. 
2024-12-10T12:05:58,411 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_115849. 
2024-12-10T12:05:58,411 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_117985. 
2024-12-10T12:05:58,412 DEBUG [hfile-prefetch-1733832355428 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1c642949999f491caefb51a05c88376d_119184. 
2024-12-10T12:05:58,412 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/1c642949999f491caefb51a05c88376d, with total blocks 57, and data blocks 56
2024-12-10T12:05:58,412 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map
2024-12-10T12:05:58,412 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/1c642949999f491caefb51a05c88376d, but only 0 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue.
2024-12-10T12:05:58,412 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/1c642949999f491caefb51a05c88376d. Will sleep 100ms and try the verification again.
2024-12-10T12:05:58,513 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/1c642949999f491caefb51a05c88376d, with total blocks 57, and data blocks 56
2024-12-10T12:05:58,513 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2285): iterating over 3 entries in the backing map
2024-12-10T12:05:58,513 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 0
2024-12-10T12:05:58,513 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 2129
2024-12-10T12:05:58,513 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 6457
2024-12-10T12:05:58,513 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/1c642949999f491caefb51a05c88376d, but only 3 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue.
2024-12-10T12:05:58,513 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/1c642949999f491caefb51a05c88376d. Will sleep 100ms and try the verification again.
2024-12-10T12:05:58,614 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/1c642949999f491caefb51a05c88376d, with total blocks 57, and data blocks 56
2024-12-10T12:05:58,614 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2285): iterating over 57 entries in the backing map
2024-12-10T12:05:58,614 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 0
2024-12-10T12:05:58,614 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 2129
2024-12-10T12:05:58,614 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 4287
2024-12-10T12:05:58,614 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 6457
2024-12-10T12:05:58,614 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 8587
2024-12-10T12:05:58,614 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 10676
2024-12-10T12:05:58,614 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 12803
2024-12-10T12:05:58,614 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 14998
2024-12-10T12:05:58,615 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 17161
2024-12-10T12:05:58,615 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 19314
2024-12-10T12:05:58,615 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 21494
2024-12-10T12:05:58,615 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 23643
2024-12-10T12:05:58,615 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 25751
2024-12-10T12:05:58,615 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 27935
2024-12-10T12:05:58,615 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 30028
2024-12-10T12:05:58,615 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 32130
2024-12-10T12:05:58,615 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 34273
2024-12-10T12:05:58,615 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 36365
2024-12-10T12:05:58,615 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 38496
2024-12-10T12:05:58,615 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 40585
2024-12-10T12:05:58,615 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 42690
2024-12-10T12:05:58,615 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 44940
2024-12-10T12:05:58,615 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 47114
2024-12-10T12:05:58,615 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 49267
2024-12-10T12:05:58,615 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 51381
2024-12-10T12:05:58,615 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 53494
2024-12-10T12:05:58,616 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 55625
2024-12-10T12:05:58,616 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 57840
2024-12-10T12:05:58,616 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 60019
2024-12-10T12:05:58,616 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 62120
2024-12-10T12:05:58,616 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 64302
2024-12-10T12:05:58,616 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 66469
2024-12-10T12:05:58,616 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 68645
2024-12-10T12:05:58,616 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 70820
2024-12-10T12:05:58,616 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 72922
2024-12-10T12:05:58,616 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 75052
2024-12-10T12:05:58,616 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 77156
2024-12-10T12:05:58,616 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 79268
2024-12-10T12:05:58,616 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 81388
2024-12-10T12:05:58,616 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 83526
2024-12-10T12:05:58,616 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 85701
2024-12-10T12:05:58,616 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 87843
2024-12-10T12:05:58,616 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 90078
2024-12-10T12:05:58,616 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 92229
2024-12-10T12:05:58,616 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 94392
2024-12-10T12:05:58,616 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 96562
2024-12-10T12:05:58,617 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 98731
2024-12-10T12:05:58,617 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 100855
2024-12-10T12:05:58,617 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 102955
2024-12-10T12:05:58,617 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 105042
2024-12-10T12:05:58,617 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 107264
2024-12-10T12:05:58,617 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 109410
2024-12-10T12:05:58,617 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 111576
2024-12-10T12:05:58,617 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 113729
2024-12-10T12:05:58,617 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 115849
2024-12-10T12:05:58,617 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 117985
2024-12-10T12:05:58,617 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2293): found block for file 1c642949999f491caefb51a05c88376d in the backing map. Acquiring read lock for offset 119184
2024-12-10T12:05:58,617 DEBUG [hfile-prefetch-1733832355428 {}] bucket.BucketCache(2305): File /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch3/1c642949999f491caefb51a05c88376d has now been fully cached.
2024-12-10T12:05:58,618 DEBUG [hfile-prefetch-1733832355428 {}] hfile.PrefetchExecutor(121): Prefetch completed for 1c642949999f491caefb51a05c88376d
2024-12-10T12:05:58,666 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk
2024-12-10T12:05:58,667 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file
2024-12-10T12:05:58,667 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format.
2024-12-10T12:05:58,667 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 57, fullycachedFiles size: 1, chunkSize: 10000
2024-12-10T12:05:58,676 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 57, fullycachedFiles size: 1
2024-12-10T12:05:58,676 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 57, fullycachedFiles size: 1, file name: bucket.persistence1733832358667
2024-12-10T12:05:58,676 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming
2024-12-10T12:05:59,492 DEBUG [Time-limited test {}] bucket.BucketCache(856): removed key 1c642949999f491caefb51a05c88376d_92229 from back map with offset lock 430080 in the evict process
2024-12-10T12:05:59,492 DEBUG [Time-limited test {}] bucket.BucketCache(768): Removing file 1c642949999f491caefb51a05c88376d for region 60b630fd-859b-fdcd-e888-62694afbc0ca
2024-12-10T12:05:59,494 INFO  [Time-limited test {}] bucket.BucketCache(1738): Disabling cache
2024-12-10T12:05:59,495 INFO  [Time-limited test {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/bucket.persistence
2024-12-10T12:05:59,495 INFO  [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false
2024-12-10T12:05:59,495 INFO  [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false
2024-12-10T12:05:59,495 INFO  [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false
2024-12-10T12:05:59,495 INFO  [Time-limited test {}] bucket.BucketCache(1768): Shutting down cache persister thread.
2024-12-10T12:05:59,495 WARN  [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread.
java.lang.InterruptedException: sleep interrupted
	at java.lang.Thread.sleep(Native Method) ~[?:?]
	at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?]
2024-12-10T12:05:59,505 DEBUG [Time-limited test {}] bucket.BucketCache(1487): Thread Time-limited test started persisting bucket cache to file
2024-12-10T12:05:59,506 DEBUG [Time-limited test {}] bucket.BucketCache(1494): Persist in new chunked persistence format.
2024-12-10T12:05:59,506 DEBUG [Time-limited test {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 56, fullycachedFiles size: 0, chunkSize: 10000
2024-12-10T12:05:59,516 DEBUG [Time-limited test {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 56, fullycachedFiles size: 0
2024-12-10T12:05:59,516 DEBUG [Time-limited test {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 56, fullycachedFiles size: 0, file name: bucket.persistence1733832359506
2024-12-10T12:05:59,516 DEBUG [Time-limited test {}] bucket.BucketCache(1509): Thread Time-limited test finished persisting bucket cache to file, renaming
2024-12-10T12:05:59,553 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-10T12:05:59,554 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-10T12:05:59,565 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestBucketCachePersister#testPrefetchListUponBlockEviction Thread=22 (was 21)
Potentially hanging thread: hfile-prefetch-1733832357489
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=290 (was 289) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=192 (was 192), ProcessCount=11 (was 11), AvailableMemoryMB=6950 (was 6797) - AvailableMemoryMB LEAK? -
2024-12-10T12:05:59,571 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestBucketCachePersister#testPrefetchPersistenceCrash Thread=22, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=192, ProcessCount=11, AvailableMemoryMB=6949
2024-12-10T12:05:59,572 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-10T12:05:59,576 INFO  [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/testPrefetchPersistenceCrash
2024-12-10T12:05:59,576 INFO  [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false
2024-12-10T12:05:59,576 INFO  [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread
2024-12-10T12:05:59,576 INFO  [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 3000
2024-12-10T12:05:59,577 INFO  [Thread-24 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file
2024-12-10T12:05:59,577 WARN  [Thread-24 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache.
2024-12-10T12:05:59,577 INFO  [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/testPrefetchPersistenceCrash, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator
2024-12-10T12:05:59,577 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-10T12:05:59,577 INFO  [Thread-24 {}] bucket.BucketAllocator(343): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size))
2024-12-10T12:05:59,578 INFO  [Thread-24 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/bucket.persistence is complete.
2024-12-10T12:05:59,587 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 314b9ceb98084064b0e795a2be28a9c8
2024-12-10T12:05:59,595 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch0/314b9ceb98084064b0e795a2be28a9c8 is 215, key is aaaaaaaaaaaaaaaaaaaaaababbbbabaa/E:tte5CE8JUzm/6774484929919482388/DeleteColumn/seqid=0
2024-12-10T12:05:59,601 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on fadcdd74f9bf442194ebba9d04753bbd
2024-12-10T12:05:59,611 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch1/fadcdd74f9bf442194ebba9d04753bbd is 224, key is aaaaaaaaaaaaaaaaaaaaaabbababbabb/V2P:/8317577373696000131/DeleteFamily/seqid=0
2024-12-10T12:05:59,612 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch0/314b9ceb98084064b0e795a2be28a9c8, delay=986 ms
2024-12-10T12:06:00,600 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_0. 
2024-12-10T12:06:00,601 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_2193. 
2024-12-10T12:06:00,601 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_4296. 
2024-12-10T12:06:00,602 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_6407. 
2024-12-10T12:06:00,602 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_8667. 
2024-12-10T12:06:00,602 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_10767. 
2024-12-10T12:06:00,603 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_12943. 
2024-12-10T12:06:00,603 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_15048. 
2024-12-10T12:06:00,604 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_17206. 
2024-12-10T12:06:00,604 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_19373. 
2024-12-10T12:06:00,605 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_21461. 
2024-12-10T12:06:00,605 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_23636. 
2024-12-10T12:06:00,605 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_25794. 
2024-12-10T12:06:00,606 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_27974. 
2024-12-10T12:06:00,606 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_30061. 
2024-12-10T12:06:00,607 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_32234. 
2024-12-10T12:06:00,607 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_34366. 
2024-12-10T12:06:00,608 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_36559. 
2024-12-10T12:06:00,608 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_38700. 
2024-12-10T12:06:00,608 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_40829. 
2024-12-10T12:06:00,609 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_42965. 
2024-12-10T12:06:00,609 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_45050. 
2024-12-10T12:06:00,610 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_47195. 
2024-12-10T12:06:00,610 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_49286. 
2024-12-10T12:06:00,610 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_51423. 
2024-12-10T12:06:00,611 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_53586. 
2024-12-10T12:06:00,611 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_55760. 
2024-12-10T12:06:00,612 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_57888. 
2024-12-10T12:06:00,612 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_60054. 
2024-12-10T12:06:00,613 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_62231. 
2024-12-10T12:06:00,613 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_64424. 
2024-12-10T12:06:00,613 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_66585. 
2024-12-10T12:06:00,614 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_68728. 
2024-12-10T12:06:00,614 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_70872. 
2024-12-10T12:06:00,615 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_72975. 
2024-12-10T12:06:00,615 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_75211. 
2024-12-10T12:06:00,615 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_77345. 
2024-12-10T12:06:00,615 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_79594. 
2024-12-10T12:06:00,615 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_81753. 
2024-12-10T12:06:00,615 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_83844. 
2024-12-10T12:06:00,616 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_85955. 
2024-12-10T12:06:00,616 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_88098. 
2024-12-10T12:06:00,616 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_90190. 
2024-12-10T12:06:00,616 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_92386. 
2024-12-10T12:06:00,616 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_94526. 
2024-12-10T12:06:00,617 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_96636. 
2024-12-10T12:06:00,617 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_98752. 
2024-12-10T12:06:00,617 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_100979. 
2024-12-10T12:06:00,617 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_103088. 
2024-12-10T12:06:00,617 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_105284. 
2024-12-10T12:06:00,617 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_107424. 
2024-12-10T12:06:00,618 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_109654. 
2024-12-10T12:06:00,618 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_111744. 
2024-12-10T12:06:00,618 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_113855. 
2024-12-10T12:06:00,618 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_116001. 
2024-12-10T12:06:00,618 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_118101. 
2024-12-10T12:06:00,618 DEBUG [hfile-prefetch-1733832354125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 314b9ceb98084064b0e795a2be28a9c8_119074. 
2024-12-10T12:06:00,619 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch0/314b9ceb98084064b0e795a2be28a9c8, with total blocks 57, and data blocks 56
2024-12-10T12:06:00,619 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map
2024-12-10T12:06:00,619 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch0/314b9ceb98084064b0e795a2be28a9c8, but only 0 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue.
2024-12-10T12:06:00,619 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch0/314b9ceb98084064b0e795a2be28a9c8. Will sleep 100ms and try the verification again.
2024-12-10T12:06:00,719 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch0/314b9ceb98084064b0e795a2be28a9c8, with total blocks 57, and data blocks 56
2024-12-10T12:06:00,719 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2285): iterating over 3 entries in the backing map
2024-12-10T12:06:00,720 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 0
2024-12-10T12:06:00,720 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 6407
2024-12-10T12:06:00,720 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 12943
2024-12-10T12:06:00,720 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch0/314b9ceb98084064b0e795a2be28a9c8, but only 3 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue.
2024-12-10T12:06:00,720 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch0/314b9ceb98084064b0e795a2be28a9c8. Will sleep 100ms and try the verification again.
2024-12-10T12:06:00,821 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch0/314b9ceb98084064b0e795a2be28a9c8, with total blocks 57, and data blocks 56
2024-12-10T12:06:00,821 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2285): iterating over 57 entries in the backing map
2024-12-10T12:06:00,821 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 0
2024-12-10T12:06:00,821 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 2193
2024-12-10T12:06:00,821 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 4296
2024-12-10T12:06:00,821 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 6407
2024-12-10T12:06:00,821 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 8667
2024-12-10T12:06:00,821 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 10767
2024-12-10T12:06:00,821 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 12943
2024-12-10T12:06:00,821 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 15048
2024-12-10T12:06:00,821 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 17206
2024-12-10T12:06:00,821 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 19373
2024-12-10T12:06:00,821 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 21461
2024-12-10T12:06:00,822 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 23636
2024-12-10T12:06:00,822 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 25794
2024-12-10T12:06:00,822 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 27974
2024-12-10T12:06:00,822 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 30061
2024-12-10T12:06:00,822 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 32234
2024-12-10T12:06:00,822 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 34366
2024-12-10T12:06:00,822 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 36559
2024-12-10T12:06:00,822 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 38700
2024-12-10T12:06:00,822 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 40829
2024-12-10T12:06:00,822 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 42965
2024-12-10T12:06:00,822 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 45050
2024-12-10T12:06:00,822 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 47195
2024-12-10T12:06:00,822 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 49286
2024-12-10T12:06:00,822 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 51423
2024-12-10T12:06:00,822 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 53586
2024-12-10T12:06:00,823 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 55760
2024-12-10T12:06:00,823 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 57888
2024-12-10T12:06:00,823 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 60054
2024-12-10T12:06:00,823 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 62231
2024-12-10T12:06:00,823 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 64424
2024-12-10T12:06:00,823 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 66585
2024-12-10T12:06:00,823 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 68728
2024-12-10T12:06:00,823 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 70872
2024-12-10T12:06:00,823 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 72975
2024-12-10T12:06:00,823 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 75211
2024-12-10T12:06:00,823 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 77345
2024-12-10T12:06:00,823 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 79594
2024-12-10T12:06:00,823 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 81753
2024-12-10T12:06:00,823 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 83844
2024-12-10T12:06:00,823 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 85955
2024-12-10T12:06:00,824 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 88098
2024-12-10T12:06:00,824 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 90190
2024-12-10T12:06:00,824 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 92386
2024-12-10T12:06:00,824 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 94526
2024-12-10T12:06:00,824 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 96636
2024-12-10T12:06:00,824 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 98752
2024-12-10T12:06:00,824 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 100979
2024-12-10T12:06:00,824 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 103088
2024-12-10T12:06:00,824 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 105284
2024-12-10T12:06:00,824 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 107424
2024-12-10T12:06:00,824 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 109654
2024-12-10T12:06:00,824 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 111744
2024-12-10T12:06:00,824 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 113855
2024-12-10T12:06:00,824 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 116001
2024-12-10T12:06:00,824 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 118101
2024-12-10T12:06:00,825 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2293): found block for file 314b9ceb98084064b0e795a2be28a9c8 in the backing map. Acquiring read lock for offset 119074
2024-12-10T12:06:00,825 DEBUG [hfile-prefetch-1733832354125 {}] bucket.BucketCache(2305): File /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch0/314b9ceb98084064b0e795a2be28a9c8 has now been fully cached.
2024-12-10T12:06:00,825 DEBUG [hfile-prefetch-1733832354125 {}] hfile.PrefetchExecutor(121): Prefetch completed for 314b9ceb98084064b0e795a2be28a9c8
2024-12-10T12:06:01,619 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch1/fadcdd74f9bf442194ebba9d04753bbd, delay=901 ms
2024-12-10T12:06:02,524 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_0. 
2024-12-10T12:06:02,525 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_2114. 
2024-12-10T12:06:02,526 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_4219. 
2024-12-10T12:06:02,527 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_6334. 
2024-12-10T12:06:02,528 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_8485. 
2024-12-10T12:06:02,529 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_10645. 
2024-12-10T12:06:02,530 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_12792. 
2024-12-10T12:06:02,531 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_14951. 
2024-12-10T12:06:02,531 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_17132. 
2024-12-10T12:06:02,531 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_19259. 
2024-12-10T12:06:02,531 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_21410. 
2024-12-10T12:06:02,532 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_23506. 
2024-12-10T12:06:02,532 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_25651. 
2024-12-10T12:06:02,532 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_27748. 
2024-12-10T12:06:02,532 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_29897. 
2024-12-10T12:06:02,532 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_31991. 
2024-12-10T12:06:02,532 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_34126. 
2024-12-10T12:06:02,533 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_36328. 
2024-12-10T12:06:02,533 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_38451. 
2024-12-10T12:06:02,533 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_40606. 
2024-12-10T12:06:02,533 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_42699. 
2024-12-10T12:06:02,533 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_44885. 
2024-12-10T12:06:02,533 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_47016. 
2024-12-10T12:06:02,534 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_49185. 
2024-12-10T12:06:02,534 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_51339. 
2024-12-10T12:06:02,534 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_53487. 
2024-12-10T12:06:02,534 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_55642. 
2024-12-10T12:06:02,534 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_57857. 
2024-12-10T12:06:02,534 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_59950. 
2024-12-10T12:06:02,535 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_62143. 
2024-12-10T12:06:02,535 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_64247. 
2024-12-10T12:06:02,535 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_66412. 
2024-12-10T12:06:02,535 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_68520. 
2024-12-10T12:06:02,535 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_70690. 
2024-12-10T12:06:02,535 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_72927. 
2024-12-10T12:06:02,536 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_75035. 
2024-12-10T12:06:02,536 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_77236. 
2024-12-10T12:06:02,536 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_79411. 
2024-12-10T12:06:02,536 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_81509. 
2024-12-10T12:06:02,536 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_83621. 
2024-12-10T12:06:02,536 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_85731. 
2024-12-10T12:06:02,537 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_87831. 
2024-12-10T12:06:02,537 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_89936. 
2024-12-10T12:06:02,537 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_92049. 
2024-12-10T12:06:02,537 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_94238. 
2024-12-10T12:06:02,537 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_96324. 
2024-12-10T12:06:02,537 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_98489. 
2024-12-10T12:06:02,538 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_100591. 
2024-12-10T12:06:02,538 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_102690. 
2024-12-10T12:06:02,538 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_104836. 
2024-12-10T12:06:02,538 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_106946. 
2024-12-10T12:06:02,538 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_109114. 
2024-12-10T12:06:02,538 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_111290. 
2024-12-10T12:06:02,539 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_113441. 
2024-12-10T12:06:02,539 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_115550. 
2024-12-10T12:06:02,539 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_117707. 
2024-12-10T12:06:02,539 DEBUG [hfile-prefetch-1733832357489 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fadcdd74f9bf442194ebba9d04753bbd_119107. 
2024-12-10T12:06:02,539 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch1/fadcdd74f9bf442194ebba9d04753bbd, with total blocks 57, and data blocks 56
2024-12-10T12:06:02,539 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2285): iterating over 57 entries in the backing map
2024-12-10T12:06:02,539 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch1/fadcdd74f9bf442194ebba9d04753bbd, but only 0 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue.
2024-12-10T12:06:02,539 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch1/fadcdd74f9bf442194ebba9d04753bbd. Will sleep 100ms and try the verification again.
2024-12-10T12:06:02,577 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk
2024-12-10T12:06:02,577 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file
2024-12-10T12:06:02,577 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format.
2024-12-10T12:06:02,577 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 57, fullycachedFiles size: 1, chunkSize: 10000
2024-12-10T12:06:02,582 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 57, fullycachedFiles size: 1
2024-12-10T12:06:02,582 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 57, fullycachedFiles size: 1, file name: bucket.persistence1733832362577
2024-12-10T12:06:02,582 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming
2024-12-10T12:06:02,640 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch1/fadcdd74f9bf442194ebba9d04753bbd, with total blocks 57, and data blocks 56
2024-12-10T12:06:02,640 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2285): iterating over 57 entries in the backing map
2024-12-10T12:06:02,640 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch1/fadcdd74f9bf442194ebba9d04753bbd, but only 0 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue.
2024-12-10T12:06:02,640 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch1/fadcdd74f9bf442194ebba9d04753bbd. Will sleep 100ms and try the verification again.
2024-12-10T12:06:02,741 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch1/fadcdd74f9bf442194ebba9d04753bbd, with total blocks 57, and data blocks 56
2024-12-10T12:06:02,741 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2285): iterating over 57 entries in the backing map
2024-12-10T12:06:02,741 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch1/fadcdd74f9bf442194ebba9d04753bbd, but only 0 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue.
2024-12-10T12:06:02,741 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch1/fadcdd74f9bf442194ebba9d04753bbd. Will sleep 100ms and try the verification again.
2024-12-10T12:06:02,842 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch1/fadcdd74f9bf442194ebba9d04753bbd, with total blocks 57, and data blocks 56
2024-12-10T12:06:02,842 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2285): iterating over 57 entries in the backing map
2024-12-10T12:06:02,842 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch1/fadcdd74f9bf442194ebba9d04753bbd, but only 0 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue.
2024-12-10T12:06:02,842 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch1/fadcdd74f9bf442194ebba9d04753bbd. Will sleep 100ms and try the verification again.
2024-12-10T12:06:02,943 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch1/fadcdd74f9bf442194ebba9d04753bbd, with total blocks 57, and data blocks 56
2024-12-10T12:06:02,943 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2285): iterating over 57 entries in the backing map
2024-12-10T12:06:02,943 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch1/fadcdd74f9bf442194ebba9d04753bbd, but only 0 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue.
2024-12-10T12:06:02,944 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch1/fadcdd74f9bf442194ebba9d04753bbd. Will sleep 100ms and try the verification again.
2024-12-10T12:06:03,044 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch1/fadcdd74f9bf442194ebba9d04753bbd, with total blocks 57, and data blocks 56
2024-12-10T12:06:03,044 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2285): iterating over 57 entries in the backing map
2024-12-10T12:06:03,044 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch1/fadcdd74f9bf442194ebba9d04753bbd, but only 0 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue.
2024-12-10T12:06:03,045 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch1/fadcdd74f9bf442194ebba9d04753bbd. Will sleep 100ms and try the verification again.
2024-12-10T12:06:03,145 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch1/fadcdd74f9bf442194ebba9d04753bbd, with total blocks 57, and data blocks 56
2024-12-10T12:06:03,145 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2285): iterating over 57 entries in the backing map
2024-12-10T12:06:03,145 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch1/fadcdd74f9bf442194ebba9d04753bbd, but only 0 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue.
2024-12-10T12:06:03,146 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch1/fadcdd74f9bf442194ebba9d04753bbd. Will sleep 100ms and try the verification again.
2024-12-10T12:06:03,246 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch1/fadcdd74f9bf442194ebba9d04753bbd, with total blocks 57, and data blocks 56
2024-12-10T12:06:03,246 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2285): iterating over 57 entries in the backing map
2024-12-10T12:06:03,246 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch1/fadcdd74f9bf442194ebba9d04753bbd, but only 0 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue.
2024-12-10T12:06:03,247 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch1/fadcdd74f9bf442194ebba9d04753bbd. Will sleep 100ms and try the verification again.
2024-12-10T12:06:03,347 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch1/fadcdd74f9bf442194ebba9d04753bbd, with total blocks 57, and data blocks 56
2024-12-10T12:06:03,347 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2285): iterating over 57 entries in the backing map
2024-12-10T12:06:03,347 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch1/fadcdd74f9bf442194ebba9d04753bbd, but only 0 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue.
2024-12-10T12:06:03,348 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch1/fadcdd74f9bf442194ebba9d04753bbd. Will sleep 100ms and try the verification again.
2024-12-10T12:06:03,448 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch1/fadcdd74f9bf442194ebba9d04753bbd, with total blocks 57, and data blocks 56
2024-12-10T12:06:03,448 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2285): iterating over 57 entries in the backing map
2024-12-10T12:06:03,448 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch1/fadcdd74f9bf442194ebba9d04753bbd, but only 0 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue.
2024-12-10T12:06:03,449 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch1/fadcdd74f9bf442194ebba9d04753bbd. Will sleep 100ms and try the verification again.
2024-12-10T12:06:03,549 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch1/fadcdd74f9bf442194ebba9d04753bbd, with total blocks 57, and data blocks 56
2024-12-10T12:06:03,549 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2285): iterating over 60 entries in the backing map
2024-12-10T12:06:03,549 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 0
2024-12-10T12:06:03,550 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 2114
2024-12-10T12:06:03,550 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 4219
2024-12-10T12:06:03,550 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch1/fadcdd74f9bf442194ebba9d04753bbd, but only 3 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue.
2024-12-10T12:06:03,550 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch1/fadcdd74f9bf442194ebba9d04753bbd. Will sleep 100ms and try the verification again.
2024-12-10T12:06:03,651 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch1/fadcdd74f9bf442194ebba9d04753bbd, with total blocks 57, and data blocks 56
2024-12-10T12:06:03,651 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2285): iterating over 114 entries in the backing map
2024-12-10T12:06:03,651 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 0
2024-12-10T12:06:03,651 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 2114
2024-12-10T12:06:03,651 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 4219
2024-12-10T12:06:03,651 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 6334
2024-12-10T12:06:03,652 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 8485
2024-12-10T12:06:03,652 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 10645
2024-12-10T12:06:03,652 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 12792
2024-12-10T12:06:03,652 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 14951
2024-12-10T12:06:03,652 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 17132
2024-12-10T12:06:03,652 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 19259
2024-12-10T12:06:03,652 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 21410
2024-12-10T12:06:03,652 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 23506
2024-12-10T12:06:03,653 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 25651
2024-12-10T12:06:03,653 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 27748
2024-12-10T12:06:03,653 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 29897
2024-12-10T12:06:03,653 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 31991
2024-12-10T12:06:03,653 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 34126
2024-12-10T12:06:03,653 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 36328
2024-12-10T12:06:03,654 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 38451
2024-12-10T12:06:03,654 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 40606
2024-12-10T12:06:03,654 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 42699
2024-12-10T12:06:03,654 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 44885
2024-12-10T12:06:03,654 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 47016
2024-12-10T12:06:03,654 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 49185
2024-12-10T12:06:03,654 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 51339
2024-12-10T12:06:03,655 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 53487
2024-12-10T12:06:03,655 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 55642
2024-12-10T12:06:03,655 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 57857
2024-12-10T12:06:03,655 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 59950
2024-12-10T12:06:03,655 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 62143
2024-12-10T12:06:03,655 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 64247
2024-12-10T12:06:03,655 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 66412
2024-12-10T12:06:03,655 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 68520
2024-12-10T12:06:03,656 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 70690
2024-12-10T12:06:03,656 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 72927
2024-12-10T12:06:03,656 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 75035
2024-12-10T12:06:03,656 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 77236
2024-12-10T12:06:03,656 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 79411
2024-12-10T12:06:03,656 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 81509
2024-12-10T12:06:03,656 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 83621
2024-12-10T12:06:03,656 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 85731
2024-12-10T12:06:03,656 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 87831
2024-12-10T12:06:03,657 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 89936
2024-12-10T12:06:03,657 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 92049
2024-12-10T12:06:03,657 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 94238
2024-12-10T12:06:03,657 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 96324
2024-12-10T12:06:03,657 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 98489
2024-12-10T12:06:03,657 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 100591
2024-12-10T12:06:03,657 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 102690
2024-12-10T12:06:03,657 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 104836
2024-12-10T12:06:03,657 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 106946
2024-12-10T12:06:03,658 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 109114
2024-12-10T12:06:03,658 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 111290
2024-12-10T12:06:03,658 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 113441
2024-12-10T12:06:03,658 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 115550
2024-12-10T12:06:03,658 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 117707
2024-12-10T12:06:03,658 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2293): found block for file fadcdd74f9bf442194ebba9d04753bbd in the backing map. Acquiring read lock for offset 119107
2024-12-10T12:06:03,659 DEBUG [hfile-prefetch-1733832357489 {}] bucket.BucketCache(2305): File /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/TestPrefetch1/fadcdd74f9bf442194ebba9d04753bbd has now been fully cached.
2024-12-10T12:06:03,659 DEBUG [hfile-prefetch-1733832357489 {}] hfile.PrefetchExecutor(121): Prefetch completed for fadcdd74f9bf442194ebba9d04753bbd
2024-12-10T12:06:05,583 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk
2024-12-10T12:06:05,583 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file
2024-12-10T12:06:05,584 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format.
2024-12-10T12:06:05,584 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 114, fullycachedFiles size: 2, chunkSize: 10000
2024-12-10T12:06:05,592 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 114, fullycachedFiles size: 2
2024-12-10T12:06:05,593 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 114, fullycachedFiles size: 2, file name: bucket.persistence1733832365583
2024-12-10T12:06:05,593 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming
2024-12-10T12:06:07,625 INFO  [Time-limited test {}] bucket.BucketCache(1738): Disabling cache
2024-12-10T12:06:07,626 INFO  [Time-limited test {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/60b630fd-859b-fdcd-e888-62694afbc0ca/bucket.persistence
2024-12-10T12:06:07,626 INFO  [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false
2024-12-10T12:06:07,626 INFO  [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false
2024-12-10T12:06:07,626 INFO  [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false
2024-12-10T12:06:07,627 INFO  [Time-limited test {}] bucket.BucketCache(1768): Shutting down cache persister thread.
2024-12-10T12:06:07,627 WARN  [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread.
java.lang.InterruptedException: sleep interrupted
	at java.lang.Thread.sleep(Native Method) ~[?:?]
	at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?]
2024-12-10T12:06:07,637 DEBUG [Time-limited test {}] bucket.BucketCache(1487): Thread Time-limited test started persisting bucket cache to file
2024-12-10T12:06:07,637 DEBUG [Time-limited test {}] bucket.BucketCache(1494): Persist in new chunked persistence format.
2024-12-10T12:06:07,637 DEBUG [Time-limited test {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 114, fullycachedFiles size: 2, chunkSize: 10000
2024-12-10T12:06:07,645 DEBUG [Time-limited test {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 114, fullycachedFiles size: 2
2024-12-10T12:06:07,645 DEBUG [Time-limited test {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 114, fullycachedFiles size: 2, file name: bucket.persistence1733832367637
2024-12-10T12:06:07,646 DEBUG [Time-limited test {}] bucket.BucketCache(1509): Thread Time-limited test finished persisting bucket cache to file, renaming
2024-12-10T12:06:07,646 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-10T12:06:07,647 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-10T12:06:07,654 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestBucketCachePersister#testPrefetchPersistenceCrash Thread=23 (was 22)
Potentially hanging thread: hfile-prefetch-1733832359613
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=292 (was 290) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=162 (was 192), ProcessCount=11 (was 11), AvailableMemoryMB=6942 (was 6949)