2024-12-11 22:56:18,045 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@57bc27f5 2024-12-11 22:56:18,061 main DEBUG Took 0.014906 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-11 22:56:18,062 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-11 22:56:18,062 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-11 22:56:18,063 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-11 22:56:18,064 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 22:56:18,071 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-11 22:56:18,083 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 22:56:18,084 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 22:56:18,084 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 22:56:18,085 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 22:56:18,085 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 22:56:18,086 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 22:56:18,086 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 22:56:18,087 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 22:56:18,087 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 22:56:18,087 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 22:56:18,088 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 22:56:18,088 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 22:56:18,089 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 22:56:18,089 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 22:56:18,089 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 22:56:18,090 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 22:56:18,090 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 22:56:18,090 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 22:56:18,091 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 22:56:18,091 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 22:56:18,091 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 22:56:18,091 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 22:56:18,092 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 22:56:18,092 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 22:56:18,092 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 22:56:18,093 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-11 22:56:18,094 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 22:56:18,095 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-11 22:56:18,097 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-11 22:56:18,097 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-11 22:56:18,098 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-11 22:56:18,099 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-11 22:56:18,106 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-11 22:56:18,109 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-11 22:56:18,110 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-11 22:56:18,111 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-11 22:56:18,111 main DEBUG createAppenders(={Console}) 2024-12-11 22:56:18,112 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@57bc27f5 initialized 2024-12-11 22:56:18,112 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@57bc27f5 2024-12-11 22:56:18,113 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@57bc27f5 OK. 2024-12-11 22:56:18,113 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-11 22:56:18,114 main DEBUG OutputStream closed 2024-12-11 22:56:18,114 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-11 22:56:18,114 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-11 22:56:18,114 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5a56cdac OK 2024-12-11 22:56:18,178 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-11 22:56:18,180 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-11 22:56:18,181 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-11 22:56:18,182 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-11 22:56:18,183 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-11 22:56:18,183 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-11 22:56:18,184 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-11 22:56:18,184 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-11 22:56:18,185 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-11 22:56:18,185 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-11 22:56:18,185 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-11 22:56:18,185 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-11 22:56:18,186 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-11 22:56:18,186 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-11 22:56:18,186 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-11 22:56:18,187 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-11 22:56:18,187 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-11 22:56:18,188 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-11 22:56:18,190 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-11 22:56:18,190 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@1608bcbd) with optional ClassLoader: null 2024-12-11 22:56:18,190 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-11 22:56:18,191 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@1608bcbd] started OK. 2024-12-11T22:56:18,203 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.bucket.TestBucketCachePersister timeout: 13 mins 2024-12-11 22:56:18,205 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-11 22:56:18,206 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-11T22:56:18,398 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d 2024-12-11T22:56:18,418 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestBucketCachePersister#testPrefetchBlockEvictionWhilePrefetchRunning Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=356, ProcessCount=11, AvailableMemoryMB=9602 2024-12-11T22:56:18,506 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-11T22:56:18,594 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T22:56:18,621 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/testPrefetchBlockEvictionWhilePrefetchRunning 2024-12-11T22:56:18,633 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-11T22:56:18,635 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-11T22:56:18,638 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 200 2024-12-11T22:56:18,639 INFO [Thread-5 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-11T22:56:18,640 WARN [Thread-5 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-11T22:56:18,645 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/testPrefetchBlockEvictionWhilePrefetchRunning, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-11T22:56:18,648 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T22:56:18,650 INFO [Thread-5 {}] bucket.BucketAllocator(343): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-11T22:56:18,650 INFO [Thread-5 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/bucket.persistence is complete. 2024-12-11T22:56:18,680 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-11T22:56:18,739 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 9533fbe6f20d4ae9b96a871e3db611e7 2024-12-11T22:56:18,824 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-11T22:56:18,856 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-11T22:56:18,892 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is 217, key is aaaaaaaaaaaaaaaaaaaaaababaababab/yjfvM:T7Dx5Ng/2951704267011085311/Put/seqid=0 2024-12-11T22:56:18,985 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7, delay=997 ms 2024-12-11T22:56:19,016 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,027 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,037 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,048 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,059 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,069 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,080 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,090 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,101 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,111 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,122 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,132 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,143 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,154 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,164 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,174 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,185 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,195 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,206 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,216 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,227 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,238 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,248 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,259 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,270 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,281 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,292 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,303 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,314 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,325 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,335 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,345 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,356 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,366 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,376 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,387 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,398 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,408 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,419 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,429 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,440 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,451 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,462 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,473 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,484 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,494 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,505 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,516 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,527 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,538 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,549 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,560 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,571 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,582 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,592 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,603 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,614 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,624 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,635 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,646 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,657 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,667 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,678 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,688 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,699 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,709 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,720 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,730 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,741 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,751 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,761 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,772 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,782 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,793 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,804 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,815 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,825 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,836 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,847 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,858 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,868 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,879 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,890 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,901 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,911 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,922 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,933 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,944 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,955 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,965 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,976 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,987 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:19,998 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:20,001 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_0. 2024-12-11T22:56:20,008 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:20,018 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:20,029 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:20,030 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_2145. 2024-12-11T22:56:20,030 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_4268. 2024-12-11T22:56:20,031 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_6369. 2024-12-11T22:56:20,031 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_8518. 2024-12-11T22:56:20,031 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_10644. 2024-12-11T22:56:20,032 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_12750. 2024-12-11T22:56:20,032 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_14841. 2024-12-11T22:56:20,033 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_16939. 2024-12-11T22:56:20,033 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_19035. 2024-12-11T22:56:20,033 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_21171. 2024-12-11T22:56:20,034 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_23362. 2024-12-11T22:56:20,034 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_25598. 2024-12-11T22:56:20,034 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_27759. 2024-12-11T22:56:20,035 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_29877. 2024-12-11T22:56:20,035 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_32008. 2024-12-11T22:56:20,035 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_34155. 2024-12-11T22:56:20,036 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_36291. 2024-12-11T22:56:20,036 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_38381. 2024-12-11T22:56:20,036 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_40533. 2024-12-11T22:56:20,037 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_42674. 2024-12-11T22:56:20,037 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_44890. 2024-12-11T22:56:20,037 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_47049. 2024-12-11T22:56:20,038 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_49226. 2024-12-11T22:56:20,038 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_51435. 2024-12-11T22:56:20,038 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_53602. 2024-12-11T22:56:20,038 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_55697. 2024-12-11T22:56:20,039 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_57817. 2024-12-11T22:56:20,039 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_59906. 2024-12-11T22:56:20,039 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:20,039 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_62032. 2024-12-11T22:56:20,040 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_64211. 2024-12-11T22:56:20,040 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_66353. 2024-12-11T22:56:20,041 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_68503. 2024-12-11T22:56:20,041 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_70593. 2024-12-11T22:56:20,041 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_72713. 2024-12-11T22:56:20,041 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_74860. 2024-12-11T22:56:20,042 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_76979. 2024-12-11T22:56:20,042 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_79167. 2024-12-11T22:56:20,042 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_81344. 2024-12-11T22:56:20,043 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_83512. 2024-12-11T22:56:20,043 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_85664. 2024-12-11T22:56:20,043 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_87780. 2024-12-11T22:56:20,044 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_89923. 2024-12-11T22:56:20,044 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_92030. 2024-12-11T22:56:20,044 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_94182. 2024-12-11T22:56:20,045 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_96297. 2024-12-11T22:56:20,045 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_98382. 2024-12-11T22:56:20,045 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_100495. 2024-12-11T22:56:20,046 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_102661. 2024-12-11T22:56:20,046 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_104891. 2024-12-11T22:56:20,046 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_107133. 2024-12-11T22:56:20,046 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_109294. 2024-12-11T22:56:20,047 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_111519. 2024-12-11T22:56:20,047 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_113632. 2024-12-11T22:56:20,047 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_115741. 2024-12-11T22:56:20,048 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_117863. 2024-12-11T22:56:20,048 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9533fbe6f20d4ae9b96a871e3db611e7_118313. 2024-12-11T22:56:20,048 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7, with total blocks 57, and data blocks 56 2024-12-11T22:56:20,049 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-11T22:56:20,049 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:20,050 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7, but only 0 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue. 2024-12-11T22:56:20,050 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7. Will sleep 100ms and try the verification again. 2024-12-11T22:56:20,060 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:20,070 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:20,080 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:20,091 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:20,101 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:20,112 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:20,115 DEBUG [Time-limited test {}] bucket.BucketCache(856): removed key 9533fbe6f20d4ae9b96a871e3db611e7_4268 from back map with offset lock 522240 in the evict process 2024-12-11T22:56:20,115 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(182): Attempted eviction for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7. Succeeded? true 2024-12-11T22:56:20,126 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:20,136 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:20,146 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:20,151 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7, with total blocks 57, and data blocks 56 2024-12-11T22:56:20,151 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2285): iterating over 2 entries in the backing map 2024-12-11T22:56:20,151 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 0 2024-12-11T22:56:20,151 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 8518 2024-12-11T22:56:20,152 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7, but only 2 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue. 2024-12-11T22:56:20,152 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7. Will sleep 100ms and try the verification again. 2024-12-11T22:56:20,157 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:20,167 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:20,178 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:20,188 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:20,199 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:20,209 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:20,220 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:20,230 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:20,239 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-11T22:56:20,239 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-11T22:56:20,240 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-11T22:56:20,240 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 56, fullycachedFiles size: 0, chunkSize: 10000 2024-12-11T22:56:20,241 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:20,251 DEBUG [Time-limited test {}] bucket.TestBucketCachePersister(173): Entered loop as prefetch for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7 is still running. 2024-12-11T22:56:20,252 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7, with total blocks 57, and data blocks 56 2024-12-11T22:56:20,253 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2285): iterating over 56 entries in the backing map 2024-12-11T22:56:20,253 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 0 2024-12-11T22:56:20,253 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 2145 2024-12-11T22:56:20,253 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 6369 2024-12-11T22:56:20,253 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 8518 2024-12-11T22:56:20,253 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 10644 2024-12-11T22:56:20,254 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 12750 2024-12-11T22:56:20,254 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 14841 2024-12-11T22:56:20,254 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 16939 2024-12-11T22:56:20,254 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 19035 2024-12-11T22:56:20,254 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 21171 2024-12-11T22:56:20,254 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 23362 2024-12-11T22:56:20,255 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 25598 2024-12-11T22:56:20,255 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 27759 2024-12-11T22:56:20,255 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 29877 2024-12-11T22:56:20,255 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 32008 2024-12-11T22:56:20,255 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 34155 2024-12-11T22:56:20,255 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 36291 2024-12-11T22:56:20,255 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 38381 2024-12-11T22:56:20,256 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 40533 2024-12-11T22:56:20,256 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 42674 2024-12-11T22:56:20,256 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 44890 2024-12-11T22:56:20,256 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 47049 2024-12-11T22:56:20,256 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 49226 2024-12-11T22:56:20,256 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 51435 2024-12-11T22:56:20,256 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 53602 2024-12-11T22:56:20,257 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 55697 2024-12-11T22:56:20,257 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 57817 2024-12-11T22:56:20,257 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 59906 2024-12-11T22:56:20,257 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 62032 2024-12-11T22:56:20,257 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 64211 2024-12-11T22:56:20,257 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 66353 2024-12-11T22:56:20,257 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 68503 2024-12-11T22:56:20,257 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 70593 2024-12-11T22:56:20,257 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 72713 2024-12-11T22:56:20,257 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 74860 2024-12-11T22:56:20,258 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 76979 2024-12-11T22:56:20,258 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 79167 2024-12-11T22:56:20,258 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 81344 2024-12-11T22:56:20,258 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 83512 2024-12-11T22:56:20,258 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 85664 2024-12-11T22:56:20,258 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 87780 2024-12-11T22:56:20,258 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 89923 2024-12-11T22:56:20,258 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 92030 2024-12-11T22:56:20,259 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 94182 2024-12-11T22:56:20,259 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 96297 2024-12-11T22:56:20,259 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 98382 2024-12-11T22:56:20,259 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 100495 2024-12-11T22:56:20,259 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 102661 2024-12-11T22:56:20,259 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 104891 2024-12-11T22:56:20,259 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 107133 2024-12-11T22:56:20,259 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 109294 2024-12-11T22:56:20,259 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 111519 2024-12-11T22:56:20,260 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 113632 2024-12-11T22:56:20,260 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 115741 2024-12-11T22:56:20,260 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 117863 2024-12-11T22:56:20,260 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 9533fbe6f20d4ae9b96a871e3db611e7 in the backing map. Acquiring read lock for offset 118313 2024-12-11T22:56:20,260 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7, but only 55 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue. 2024-12-11T22:56:20,260 INFO [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2331): We found only 55 data blocks cached from a total of 56 for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/9533fbe6f20d4ae9b96a871e3db611e7, but no blocks pending caching. Maybe cache is full or evictions happened concurrently to cache prefetch. 2024-12-11T22:56:20,261 DEBUG [hfile-prefetch-1733957778986 {}] hfile.PrefetchExecutor(121): Prefetch completed for 9533fbe6f20d4ae9b96a871e3db611e7 2024-12-11T22:56:20,261 INFO [Time-limited test {}] bucket.BucketCache(1738): Disabling cache 2024-12-11T22:56:20,262 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-11T22:56:20,262 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-11T22:56:20,262 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-11T22:56:20,263 INFO [Time-limited test {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/bucket.persistence 2024-12-11T22:56:20,263 INFO [Time-limited test {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-11T22:56:20,316 ERROR [bucket-cache-persister {}] bucket.PersistentIOEngine(78): Calculating checksum failed, because of java.io.InterruptedIOException: java.lang.InterruptedException at org.apache.hadoop.util.Shell.runCommand(Shell.java:1071) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.getFileSize(PersistentIOEngine.java:94) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.calculateChecksum(PersistentIOEngine.java:71) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.toPB(BucketProtoUtils.java:61) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.serializeAsPB(BucketProtoUtils.java:75) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistChunkedBackingMap(BucketCache.java:1678) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1496) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] Caused by: java.lang.InterruptedException at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at java.lang.ProcessImpl.waitFor(ProcessImpl.java:434) ~[?:?] at org.apache.hadoop.util.Shell.runCommand(Shell.java:1061) ~[hadoop-common-3.4.1.jar:?] ... 9 more 2024-12-11T22:56:20,325 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 56, fullycachedFiles size: 0 2024-12-11T22:56:20,325 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 56, fullycachedFiles size: 0, file name: bucket.persistence1733957780239 2024-12-11T22:56:20,326 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-11T22:56:20,326 INFO [bucket-cache-persister {}] bucket.BucketCachePersister(62): Finishing cache persister thread. 2024-12-11T22:56:20,334 DEBUG [Time-limited test {}] bucket.BucketCache(1487): Thread Time-limited test started persisting bucket cache to file 2024-12-11T22:56:20,335 DEBUG [Time-limited test {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-11T22:56:20,335 DEBUG [Time-limited test {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 56, fullycachedFiles size: 0, chunkSize: 10000 2024-12-11T22:56:20,341 DEBUG [Time-limited test {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 56, fullycachedFiles size: 0 2024-12-11T22:56:20,341 DEBUG [Time-limited test {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 56, fullycachedFiles size: 0, file name: bucket.persistence1733957780334 2024-12-11T22:56:20,341 DEBUG [Time-limited test {}] bucket.BucketCache(1509): Thread Time-limited test finished persisting bucket cache to file, renaming 2024-12-11T22:56:20,341 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T22:56:20,342 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T22:56:20,345 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T22:56:20,351 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestBucketCachePersister#testPrefetchBlockEvictionWhilePrefetchRunning Thread=20 (was 12) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: hfile-prefetch-1733957778986 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) - Thread LEAK? -, OpenFileDescriptor=288 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=359 (was 356) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9553 (was 9602) 2024-12-11T22:56:20,355 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestBucketCachePersister#testPrefetchPersistenceCrashNegative Thread=20, OpenFileDescriptor=288, MaxFileDescriptor=1048576, SystemLoadAverage=359, ProcessCount=11, AvailableMemoryMB=9553 2024-12-11T22:56:20,356 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T22:56:20,358 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/testPrefetchPersistenceCrashNegative 2024-12-11T22:56:20,358 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-11T22:56:20,359 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-11T22:56:20,359 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 9223372036854775807 2024-12-11T22:56:20,359 INFO [Thread-13 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-11T22:56:20,359 WARN [Thread-13 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-11T22:56:20,359 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/testPrefetchPersistenceCrashNegative, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-11T22:56:20,360 INFO [Thread-13 {}] bucket.BucketAllocator(343): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-11T22:56:20,360 INFO [Thread-13 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/bucket.persistence is complete. 2024-12-11T22:56:20,360 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T22:56:20,365 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 54a3b3f8714048ce9991b452acf73594 2024-12-11T22:56:20,375 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch2/54a3b3f8714048ce9991b452acf73594 is 276, key is aaaaaaaaaaaaaaaaaaaaaaaabaaaabab/Rm2:o727h7pRv/4607676604623669284/Put/seqid=0 2024-12-11T22:56:20,377 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch2/54a3b3f8714048ce9991b452acf73594, delay=924 ms 2024-12-11T22:56:21,305 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_0. 2024-12-11T22:56:21,307 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_2167. 2024-12-11T22:56:21,309 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_4256. 2024-12-11T22:56:21,310 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_6398. 2024-12-11T22:56:21,311 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_8591. 2024-12-11T22:56:21,313 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_10723. 2024-12-11T22:56:21,314 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_12846. 2024-12-11T22:56:21,315 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_15003. 2024-12-11T22:56:21,315 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_17126. 2024-12-11T22:56:21,315 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_19228. 2024-12-11T22:56:21,316 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_21454. 2024-12-11T22:56:21,316 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_23719. 2024-12-11T22:56:21,316 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_25827. 2024-12-11T22:56:21,317 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_28058. 2024-12-11T22:56:21,317 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_30169. 2024-12-11T22:56:21,318 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_32308. 2024-12-11T22:56:21,318 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_34437. 2024-12-11T22:56:21,318 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_36531. 2024-12-11T22:56:21,319 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_38684. 2024-12-11T22:56:21,319 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_40772. 2024-12-11T22:56:21,319 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_42948. 2024-12-11T22:56:21,320 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_45143. 2024-12-11T22:56:21,320 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_47258. 2024-12-11T22:56:21,320 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_49434. 2024-12-11T22:56:21,321 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_51622. 2024-12-11T22:56:21,321 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_53765. 2024-12-11T22:56:21,322 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_56006. 2024-12-11T22:56:21,322 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_58108. 2024-12-11T22:56:21,322 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_60214. 2024-12-11T22:56:21,323 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_62422. 2024-12-11T22:56:21,323 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_64544. 2024-12-11T22:56:21,323 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_66722. 2024-12-11T22:56:21,324 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_68850. 2024-12-11T22:56:21,324 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_70987. 2024-12-11T22:56:21,325 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_73177. 2024-12-11T22:56:21,325 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_75264. 2024-12-11T22:56:21,325 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_77410. 2024-12-11T22:56:21,326 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_79529. 2024-12-11T22:56:21,326 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_81678. 2024-12-11T22:56:21,326 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_83845. 2024-12-11T22:56:21,327 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_86005. 2024-12-11T22:56:21,327 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_88108. 2024-12-11T22:56:21,327 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_90296. 2024-12-11T22:56:21,328 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_92404. 2024-12-11T22:56:21,328 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_94633. 2024-12-11T22:56:21,328 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_96758. 2024-12-11T22:56:21,329 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_98863. 2024-12-11T22:56:21,329 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_101069. 2024-12-11T22:56:21,329 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_103163. 2024-12-11T22:56:21,329 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_105273. 2024-12-11T22:56:21,330 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_107447. 2024-12-11T22:56:21,330 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_109604. 2024-12-11T22:56:21,330 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_111789. 2024-12-11T22:56:21,330 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_114011. 2024-12-11T22:56:21,331 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_116140. 2024-12-11T22:56:21,331 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_118227. 2024-12-11T22:56:21,331 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 54a3b3f8714048ce9991b452acf73594_119978. 2024-12-11T22:56:21,331 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch2/54a3b3f8714048ce9991b452acf73594, with total blocks 57, and data blocks 56 2024-12-11T22:56:21,332 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-11T22:56:21,332 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch2/54a3b3f8714048ce9991b452acf73594, but only 0 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue. 2024-12-11T22:56:21,332 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch2/54a3b3f8714048ce9991b452acf73594. Will sleep 100ms and try the verification again. 2024-12-11T22:56:21,432 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch2/54a3b3f8714048ce9991b452acf73594, with total blocks 57, and data blocks 56 2024-12-11T22:56:21,432 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2285): iterating over 2 entries in the backing map 2024-12-11T22:56:21,432 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 0 2024-12-11T22:56:21,432 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 2167 2024-12-11T22:56:21,433 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch2/54a3b3f8714048ce9991b452acf73594, but only 2 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue. 2024-12-11T22:56:21,433 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch2/54a3b3f8714048ce9991b452acf73594. Will sleep 100ms and try the verification again. 2024-12-11T22:56:21,533 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch2/54a3b3f8714048ce9991b452acf73594, with total blocks 57, and data blocks 56 2024-12-11T22:56:21,533 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2285): iterating over 3 entries in the backing map 2024-12-11T22:56:21,533 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 0 2024-12-11T22:56:21,534 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 2167 2024-12-11T22:56:21,534 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 4256 2024-12-11T22:56:21,534 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch2/54a3b3f8714048ce9991b452acf73594, but only 3 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue. 2024-12-11T22:56:21,534 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch2/54a3b3f8714048ce9991b452acf73594. Will sleep 100ms and try the verification again. 2024-12-11T22:56:21,634 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch2/54a3b3f8714048ce9991b452acf73594, with total blocks 57, and data blocks 56 2024-12-11T22:56:21,635 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2285): iterating over 36 entries in the backing map 2024-12-11T22:56:21,635 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 0 2024-12-11T22:56:21,635 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 2167 2024-12-11T22:56:21,635 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 4256 2024-12-11T22:56:21,635 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 10723 2024-12-11T22:56:21,635 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 12846 2024-12-11T22:56:21,635 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 15003 2024-12-11T22:56:21,635 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 19228 2024-12-11T22:56:21,635 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 21454 2024-12-11T22:56:21,635 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 23719 2024-12-11T22:56:21,635 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 25827 2024-12-11T22:56:21,636 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 30169 2024-12-11T22:56:21,636 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 32308 2024-12-11T22:56:21,636 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 34437 2024-12-11T22:56:21,636 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 36531 2024-12-11T22:56:21,636 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 42948 2024-12-11T22:56:21,636 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 49434 2024-12-11T22:56:21,636 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 51622 2024-12-11T22:56:21,636 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 58108 2024-12-11T22:56:21,636 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 60214 2024-12-11T22:56:21,636 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 62422 2024-12-11T22:56:21,636 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 68850 2024-12-11T22:56:21,636 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 70987 2024-12-11T22:56:21,637 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 73177 2024-12-11T22:56:21,637 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 75264 2024-12-11T22:56:21,637 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 77410 2024-12-11T22:56:21,637 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 81678 2024-12-11T22:56:21,637 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 83845 2024-12-11T22:56:21,637 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 86005 2024-12-11T22:56:21,637 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 88108 2024-12-11T22:56:21,637 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 92404 2024-12-11T22:56:21,637 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 94633 2024-12-11T22:56:21,637 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 98863 2024-12-11T22:56:21,637 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 105273 2024-12-11T22:56:21,637 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 111789 2024-12-11T22:56:21,638 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 116140 2024-12-11T22:56:21,638 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 118227 2024-12-11T22:56:21,638 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch2/54a3b3f8714048ce9991b452acf73594, but only 36 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue. 2024-12-11T22:56:21,638 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch2/54a3b3f8714048ce9991b452acf73594. Will sleep 100ms and try the verification again. 2024-12-11T22:56:21,738 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch2/54a3b3f8714048ce9991b452acf73594, with total blocks 57, and data blocks 56 2024-12-11T22:56:21,738 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2285): iterating over 36 entries in the backing map 2024-12-11T22:56:21,739 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 0 2024-12-11T22:56:21,739 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 2167 2024-12-11T22:56:21,739 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 4256 2024-12-11T22:56:21,739 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 10723 2024-12-11T22:56:21,740 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 12846 2024-12-11T22:56:21,740 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 15003 2024-12-11T22:56:21,740 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 19228 2024-12-11T22:56:21,740 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 21454 2024-12-11T22:56:21,740 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 23719 2024-12-11T22:56:21,741 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 25827 2024-12-11T22:56:21,741 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 30169 2024-12-11T22:56:21,741 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 32308 2024-12-11T22:56:21,741 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 34437 2024-12-11T22:56:21,742 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 36531 2024-12-11T22:56:21,742 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 42948 2024-12-11T22:56:21,742 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 49434 2024-12-11T22:56:21,742 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 51622 2024-12-11T22:56:21,742 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 58108 2024-12-11T22:56:21,743 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 60214 2024-12-11T22:56:21,743 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 62422 2024-12-11T22:56:21,743 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 68850 2024-12-11T22:56:21,743 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 70987 2024-12-11T22:56:21,744 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 73177 2024-12-11T22:56:21,744 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 75264 2024-12-11T22:56:21,744 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 77410 2024-12-11T22:56:21,744 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 81678 2024-12-11T22:56:21,744 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 83845 2024-12-11T22:56:21,745 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 86005 2024-12-11T22:56:21,745 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 88108 2024-12-11T22:56:21,745 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 92404 2024-12-11T22:56:21,746 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 94633 2024-12-11T22:56:21,746 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 98863 2024-12-11T22:56:21,746 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 105273 2024-12-11T22:56:21,746 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 111789 2024-12-11T22:56:21,746 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 116140 2024-12-11T22:56:21,747 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 118227 2024-12-11T22:56:21,747 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch2/54a3b3f8714048ce9991b452acf73594, but only 36 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue. 2024-12-11T22:56:21,747 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch2/54a3b3f8714048ce9991b452acf73594. Will sleep 100ms and try the verification again. 2024-12-11T22:56:21,847 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch2/54a3b3f8714048ce9991b452acf73594, with total blocks 57, and data blocks 56 2024-12-11T22:56:21,848 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2285): iterating over 57 entries in the backing map 2024-12-11T22:56:21,848 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 0 2024-12-11T22:56:21,848 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 2167 2024-12-11T22:56:21,848 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 4256 2024-12-11T22:56:21,849 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 6398 2024-12-11T22:56:21,849 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 8591 2024-12-11T22:56:21,849 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 10723 2024-12-11T22:56:21,849 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 12846 2024-12-11T22:56:21,849 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 15003 2024-12-11T22:56:21,849 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 17126 2024-12-11T22:56:21,849 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 19228 2024-12-11T22:56:21,850 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 21454 2024-12-11T22:56:21,850 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 23719 2024-12-11T22:56:21,850 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 25827 2024-12-11T22:56:21,850 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 28058 2024-12-11T22:56:21,850 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 30169 2024-12-11T22:56:21,850 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 32308 2024-12-11T22:56:21,851 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 34437 2024-12-11T22:56:21,851 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 36531 2024-12-11T22:56:21,851 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 38684 2024-12-11T22:56:21,851 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 40772 2024-12-11T22:56:21,851 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 42948 2024-12-11T22:56:21,851 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 45143 2024-12-11T22:56:21,852 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 47258 2024-12-11T22:56:21,852 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 49434 2024-12-11T22:56:21,852 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 51622 2024-12-11T22:56:21,852 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 53765 2024-12-11T22:56:21,852 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 56006 2024-12-11T22:56:21,852 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 58108 2024-12-11T22:56:21,853 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 60214 2024-12-11T22:56:21,853 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 62422 2024-12-11T22:56:21,853 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 64544 2024-12-11T22:56:21,853 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 66722 2024-12-11T22:56:21,853 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 68850 2024-12-11T22:56:21,853 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 70987 2024-12-11T22:56:21,853 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 73177 2024-12-11T22:56:21,854 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 75264 2024-12-11T22:56:21,854 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 77410 2024-12-11T22:56:21,854 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 79529 2024-12-11T22:56:21,854 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 81678 2024-12-11T22:56:21,854 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 83845 2024-12-11T22:56:21,854 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 86005 2024-12-11T22:56:21,854 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 88108 2024-12-11T22:56:21,855 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 90296 2024-12-11T22:56:21,855 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 92404 2024-12-11T22:56:21,855 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 94633 2024-12-11T22:56:21,855 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 96758 2024-12-11T22:56:21,855 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 98863 2024-12-11T22:56:21,855 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 101069 2024-12-11T22:56:21,856 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 103163 2024-12-11T22:56:21,856 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 105273 2024-12-11T22:56:21,856 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 107447 2024-12-11T22:56:21,856 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 109604 2024-12-11T22:56:21,856 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 111789 2024-12-11T22:56:21,856 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 114011 2024-12-11T22:56:21,856 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 116140 2024-12-11T22:56:21,856 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 118227 2024-12-11T22:56:21,856 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 54a3b3f8714048ce9991b452acf73594 in the backing map. Acquiring read lock for offset 119978 2024-12-11T22:56:21,856 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2305): File /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch2/54a3b3f8714048ce9991b452acf73594 has now been fully cached. 2024-12-11T22:56:21,857 DEBUG [hfile-prefetch-1733957778986 {}] hfile.PrefetchExecutor(121): Prefetch completed for 54a3b3f8714048ce9991b452acf73594 2024-12-11T22:56:22,383 INFO [Time-limited test {}] bucket.BucketCache(1738): Disabling cache 2024-12-11T22:56:22,384 INFO [Time-limited test {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/bucket.persistence 2024-12-11T22:56:22,384 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-11T22:56:22,384 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-11T22:56:22,384 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-11T22:56:22,384 INFO [Time-limited test {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-11T22:56:22,384 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-11T22:56:22,394 DEBUG [Time-limited test {}] bucket.BucketCache(1487): Thread Time-limited test started persisting bucket cache to file 2024-12-11T22:56:22,395 DEBUG [Time-limited test {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-11T22:56:22,395 DEBUG [Time-limited test {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 57, fullycachedFiles size: 1, chunkSize: 10000 2024-12-11T22:56:22,405 DEBUG [Time-limited test {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 57, fullycachedFiles size: 1 2024-12-11T22:56:22,405 DEBUG [Time-limited test {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 57, fullycachedFiles size: 1, file name: bucket.persistence1733957782394 2024-12-11T22:56:22,405 DEBUG [Time-limited test {}] bucket.BucketCache(1509): Thread Time-limited test finished persisting bucket cache to file, renaming 2024-12-11T22:56:22,405 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T22:56:22,406 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T22:56:22,410 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestBucketCachePersister#testPrefetchPersistenceCrashNegative Thread=21 (was 20) Potentially hanging thread: hfile-prefetch-1733957780377 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=289 (was 288) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=359 (was 359), ProcessCount=11 (was 11), AvailableMemoryMB=9550 (was 9553) 2024-12-11T22:56:22,415 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestBucketCachePersister#testPrefetchListUponBlockEviction Thread=21, OpenFileDescriptor=289, MaxFileDescriptor=1048576, SystemLoadAverage=359, ProcessCount=11, AvailableMemoryMB=9550 2024-12-11T22:56:22,416 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T22:56:22,418 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/testPrefetchListUponBlockEviction 2024-12-11T22:56:22,419 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-11T22:56:22,419 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-11T22:56:22,419 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 200 2024-12-11T22:56:22,419 INFO [Thread-18 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-11T22:56:22,419 WARN [Thread-18 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-11T22:56:22,420 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/testPrefetchListUponBlockEviction, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-11T22:56:22,420 INFO [Thread-18 {}] bucket.BucketAllocator(343): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-11T22:56:22,420 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T22:56:22,420 INFO [Thread-18 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/bucket.persistence is complete. 2024-12-11T22:56:22,427 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 06b1451a42144e06ae222bba148bcf4f 2024-12-11T22:56:22,436 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/06b1451a42144e06ae222bba148bcf4f is 257, key is aaaaaaaaaaaaaaaaaaaaaaaabababbbb/k:4kzDTZ7pgDTsYBa/6688526785340523934/DeleteFamilyVersion/seqid=0 2024-12-11T22:56:22,438 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/06b1451a42144e06ae222bba148bcf4f, delay=906 ms 2024-12-11T22:56:22,620 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-11T22:56:22,620 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-11T22:56:22,621 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-11T22:56:22,621 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-11T22:56:22,629 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-11T22:56:22,630 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733957782620 2024-12-11T22:56:22,630 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-11T22:56:23,348 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_0. 2024-12-11T22:56:23,349 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_2164. 2024-12-11T22:56:23,350 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_4347. 2024-12-11T22:56:23,351 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_6539. 2024-12-11T22:56:23,352 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_8632. 2024-12-11T22:56:23,352 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_10798. 2024-12-11T22:56:23,352 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_12974. 2024-12-11T22:56:23,353 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_15187. 2024-12-11T22:56:23,353 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_17303. 2024-12-11T22:56:23,354 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_19458. 2024-12-11T22:56:23,354 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_21581. 2024-12-11T22:56:23,354 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_23739. 2024-12-11T22:56:23,355 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_25902. 2024-12-11T22:56:23,355 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_28092. 2024-12-11T22:56:23,355 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_30264. 2024-12-11T22:56:23,356 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_32349. 2024-12-11T22:56:23,356 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_34503. 2024-12-11T22:56:23,357 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_36614. 2024-12-11T22:56:23,357 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_38740. 2024-12-11T22:56:23,357 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_40963. 2024-12-11T22:56:23,358 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_43117. 2024-12-11T22:56:23,358 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_45259. 2024-12-11T22:56:23,358 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_47345. 2024-12-11T22:56:23,359 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_49454. 2024-12-11T22:56:23,359 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_51662. 2024-12-11T22:56:23,359 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_53868. 2024-12-11T22:56:23,360 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_56021. 2024-12-11T22:56:23,360 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_58201. 2024-12-11T22:56:23,360 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_60340. 2024-12-11T22:56:23,361 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_62490. 2024-12-11T22:56:23,361 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_64681. 2024-12-11T22:56:23,361 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_66811. 2024-12-11T22:56:23,361 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_68957. 2024-12-11T22:56:23,361 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_71088. 2024-12-11T22:56:23,362 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_73187. 2024-12-11T22:56:23,362 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_75369. 2024-12-11T22:56:23,362 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_77544. 2024-12-11T22:56:23,362 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_79721. 2024-12-11T22:56:23,363 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_81849. 2024-12-11T22:56:23,363 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_83978. 2024-12-11T22:56:23,363 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_86094. 2024-12-11T22:56:23,363 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_88271. 2024-12-11T22:56:23,363 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_90428. 2024-12-11T22:56:23,364 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_92580. 2024-12-11T22:56:23,364 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_94798. 2024-12-11T22:56:23,364 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_96919. 2024-12-11T22:56:23,364 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_99125. 2024-12-11T22:56:23,364 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_101301. 2024-12-11T22:56:23,365 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_103441. 2024-12-11T22:56:23,365 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_105611. 2024-12-11T22:56:23,365 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_107737. 2024-12-11T22:56:23,365 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_109870. 2024-12-11T22:56:23,365 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_112076. 2024-12-11T22:56:23,366 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_114208. 2024-12-11T22:56:23,366 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_116356. 2024-12-11T22:56:23,366 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_118582. 2024-12-11T22:56:23,366 DEBUG [hfile-prefetch-1733957780377 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 06b1451a42144e06ae222bba148bcf4f_119627. 2024-12-11T22:56:23,366 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/06b1451a42144e06ae222bba148bcf4f, with total blocks 57, and data blocks 56 2024-12-11T22:56:23,367 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-11T22:56:23,367 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/06b1451a42144e06ae222bba148bcf4f, but only 0 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue. 2024-12-11T22:56:23,367 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/06b1451a42144e06ae222bba148bcf4f. Will sleep 100ms and try the verification again. 2024-12-11T22:56:23,467 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/06b1451a42144e06ae222bba148bcf4f, with total blocks 57, and data blocks 56 2024-12-11T22:56:23,467 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-11T22:56:23,467 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/06b1451a42144e06ae222bba148bcf4f, but only 0 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue. 2024-12-11T22:56:23,467 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/06b1451a42144e06ae222bba148bcf4f. Will sleep 100ms and try the verification again. 2024-12-11T22:56:23,567 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/06b1451a42144e06ae222bba148bcf4f, with total blocks 57, and data blocks 56 2024-12-11T22:56:23,568 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2285): iterating over 2 entries in the backing map 2024-12-11T22:56:23,568 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 0 2024-12-11T22:56:23,568 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 2164 2024-12-11T22:56:23,568 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/06b1451a42144e06ae222bba148bcf4f, but only 2 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue. 2024-12-11T22:56:23,568 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/06b1451a42144e06ae222bba148bcf4f. Will sleep 100ms and try the verification again. 2024-12-11T22:56:23,631 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-11T22:56:23,631 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-11T22:56:23,631 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-11T22:56:23,631 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 3, fullycachedFiles size: 0, chunkSize: 10000 2024-12-11T22:56:23,636 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 3, fullycachedFiles size: 0 2024-12-11T22:56:23,636 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 3, fullycachedFiles size: 0, file name: bucket.persistence1733957783631 2024-12-11T22:56:23,636 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-11T22:56:23,668 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/06b1451a42144e06ae222bba148bcf4f, with total blocks 57, and data blocks 56 2024-12-11T22:56:23,668 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2285): iterating over 39 entries in the backing map 2024-12-11T22:56:23,668 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 0 2024-12-11T22:56:23,668 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 2164 2024-12-11T22:56:23,668 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 4347 2024-12-11T22:56:23,668 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 6539 2024-12-11T22:56:23,669 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 8632 2024-12-11T22:56:23,669 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 10798 2024-12-11T22:56:23,669 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 15187 2024-12-11T22:56:23,669 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 19458 2024-12-11T22:56:23,669 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 23739 2024-12-11T22:56:23,669 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 25902 2024-12-11T22:56:23,669 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 28092 2024-12-11T22:56:23,669 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 30264 2024-12-11T22:56:23,669 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 32349 2024-12-11T22:56:23,669 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 34503 2024-12-11T22:56:23,669 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 38740 2024-12-11T22:56:23,669 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 40963 2024-12-11T22:56:23,669 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 43117 2024-12-11T22:56:23,669 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 45259 2024-12-11T22:56:23,669 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 53868 2024-12-11T22:56:23,669 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 58201 2024-12-11T22:56:23,669 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 60340 2024-12-11T22:56:23,669 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 62490 2024-12-11T22:56:23,669 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 64681 2024-12-11T22:56:23,670 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 66811 2024-12-11T22:56:23,670 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 71088 2024-12-11T22:56:23,670 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 75369 2024-12-11T22:56:23,670 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 77544 2024-12-11T22:56:23,670 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 81849 2024-12-11T22:56:23,670 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 86094 2024-12-11T22:56:23,670 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 92580 2024-12-11T22:56:23,670 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 94798 2024-12-11T22:56:23,670 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 96919 2024-12-11T22:56:23,670 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 101301 2024-12-11T22:56:23,670 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 103441 2024-12-11T22:56:23,670 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 107737 2024-12-11T22:56:23,670 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 109870 2024-12-11T22:56:23,670 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 114208 2024-12-11T22:56:23,670 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 116356 2024-12-11T22:56:23,670 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 118582 2024-12-11T22:56:23,670 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/06b1451a42144e06ae222bba148bcf4f, but only 39 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue. 2024-12-11T22:56:23,670 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/06b1451a42144e06ae222bba148bcf4f. Will sleep 100ms and try the verification again. 2024-12-11T22:56:23,771 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/06b1451a42144e06ae222bba148bcf4f, with total blocks 57, and data blocks 56 2024-12-11T22:56:23,771 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2285): iterating over 57 entries in the backing map 2024-12-11T22:56:23,771 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 0 2024-12-11T22:56:23,771 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 2164 2024-12-11T22:56:23,771 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 4347 2024-12-11T22:56:23,771 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 6539 2024-12-11T22:56:23,772 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 8632 2024-12-11T22:56:23,772 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 10798 2024-12-11T22:56:23,772 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 12974 2024-12-11T22:56:23,772 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 15187 2024-12-11T22:56:23,772 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 17303 2024-12-11T22:56:23,772 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 19458 2024-12-11T22:56:23,772 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 21581 2024-12-11T22:56:23,772 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 23739 2024-12-11T22:56:23,773 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 25902 2024-12-11T22:56:23,773 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 28092 2024-12-11T22:56:23,773 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 30264 2024-12-11T22:56:23,773 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 32349 2024-12-11T22:56:23,773 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 34503 2024-12-11T22:56:23,773 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 36614 2024-12-11T22:56:23,773 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 38740 2024-12-11T22:56:23,773 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 40963 2024-12-11T22:56:23,774 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 43117 2024-12-11T22:56:23,774 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 45259 2024-12-11T22:56:23,774 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 47345 2024-12-11T22:56:23,774 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 49454 2024-12-11T22:56:23,774 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 51662 2024-12-11T22:56:23,774 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 53868 2024-12-11T22:56:23,774 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 56021 2024-12-11T22:56:23,775 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 58201 2024-12-11T22:56:23,775 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 60340 2024-12-11T22:56:23,775 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 62490 2024-12-11T22:56:23,775 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 64681 2024-12-11T22:56:23,775 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 66811 2024-12-11T22:56:23,775 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 68957 2024-12-11T22:56:23,775 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 71088 2024-12-11T22:56:23,775 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 73187 2024-12-11T22:56:23,775 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 75369 2024-12-11T22:56:23,776 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 77544 2024-12-11T22:56:23,776 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 79721 2024-12-11T22:56:23,776 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 81849 2024-12-11T22:56:23,776 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 83978 2024-12-11T22:56:23,776 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 86094 2024-12-11T22:56:23,776 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 88271 2024-12-11T22:56:23,776 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 90428 2024-12-11T22:56:23,776 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 92580 2024-12-11T22:56:23,776 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 94798 2024-12-11T22:56:23,777 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 96919 2024-12-11T22:56:23,777 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 99125 2024-12-11T22:56:23,777 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 101301 2024-12-11T22:56:23,777 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 103441 2024-12-11T22:56:23,777 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 105611 2024-12-11T22:56:23,777 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 107737 2024-12-11T22:56:23,777 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 109870 2024-12-11T22:56:23,777 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 112076 2024-12-11T22:56:23,777 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 114208 2024-12-11T22:56:23,777 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 116356 2024-12-11T22:56:23,777 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 118582 2024-12-11T22:56:23,777 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2293): found block for file 06b1451a42144e06ae222bba148bcf4f in the backing map. Acquiring read lock for offset 119627 2024-12-11T22:56:23,778 DEBUG [hfile-prefetch-1733957780377 {}] bucket.BucketCache(2305): File /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch3/06b1451a42144e06ae222bba148bcf4f has now been fully cached. 2024-12-11T22:56:23,778 DEBUG [hfile-prefetch-1733957780377 {}] hfile.PrefetchExecutor(121): Prefetch completed for 06b1451a42144e06ae222bba148bcf4f 2024-12-11T22:56:23,837 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-11T22:56:23,837 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-11T22:56:23,837 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-11T22:56:23,837 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 57, fullycachedFiles size: 1, chunkSize: 10000 2024-12-11T22:56:23,843 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 57, fullycachedFiles size: 1 2024-12-11T22:56:23,843 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 57, fullycachedFiles size: 1, file name: bucket.persistence1733957783837 2024-12-11T22:56:23,843 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-11T22:56:24,442 DEBUG [Time-limited test {}] bucket.BucketCache(856): removed key 06b1451a42144e06ae222bba148bcf4f_15187 from back map with offset lock 503808 in the evict process 2024-12-11T22:56:24,442 DEBUG [Time-limited test {}] bucket.BucketCache(768): Removing file 06b1451a42144e06ae222bba148bcf4f for region 4f9723e8-69bc-fede-3ddb-eb4d59f53f1d 2024-12-11T22:56:24,444 INFO [Time-limited test {}] bucket.BucketCache(1738): Disabling cache 2024-12-11T22:56:24,444 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-11T22:56:24,444 INFO [Time-limited test {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/bucket.persistence 2024-12-11T22:56:24,445 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-11T22:56:24,445 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-11T22:56:24,445 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-11T22:56:24,445 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-11T22:56:24,445 INFO [Time-limited test {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-11T22:56:24,445 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-11T22:56:24,446 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 56, fullycachedFiles size: 0, chunkSize: 10000 2024-12-11T22:56:24,451 ERROR [bucket-cache-persister {}] bucket.PersistentIOEngine(78): Calculating checksum failed, because of java.io.InterruptedIOException: java.lang.InterruptedException at org.apache.hadoop.util.Shell.runCommand(Shell.java:1071) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.getFileSize(PersistentIOEngine.java:94) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.calculateChecksum(PersistentIOEngine.java:71) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.toPB(BucketProtoUtils.java:61) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.serializeAsPB(BucketProtoUtils.java:75) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistChunkedBackingMap(BucketCache.java:1678) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1496) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] Caused by: java.lang.InterruptedException at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at java.lang.ProcessImpl.waitFor(ProcessImpl.java:434) ~[?:?] at org.apache.hadoop.util.Shell.runCommand(Shell.java:1061) ~[hadoop-common-3.4.1.jar:?] ... 9 more 2024-12-11T22:56:24,452 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 56, fullycachedFiles size: 0 2024-12-11T22:56:24,453 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 56, fullycachedFiles size: 0, file name: bucket.persistence1733957784445 2024-12-11T22:56:24,453 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-11T22:56:24,484 INFO [bucket-cache-persister {}] bucket.BucketCachePersister(62): Finishing cache persister thread. 2024-12-11T22:56:24,486 DEBUG [Time-limited test {}] bucket.BucketCache(1487): Thread Time-limited test started persisting bucket cache to file 2024-12-11T22:56:24,486 DEBUG [Time-limited test {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-11T22:56:24,487 DEBUG [Time-limited test {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 56, fullycachedFiles size: 0, chunkSize: 10000 2024-12-11T22:56:24,494 DEBUG [Time-limited test {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 56, fullycachedFiles size: 0 2024-12-11T22:56:24,494 DEBUG [Time-limited test {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 56, fullycachedFiles size: 0, file name: bucket.persistence1733957784486 2024-12-11T22:56:24,494 DEBUG [Time-limited test {}] bucket.BucketCache(1509): Thread Time-limited test finished persisting bucket cache to file, renaming 2024-12-11T22:56:24,495 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T22:56:24,497 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T22:56:24,503 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestBucketCachePersister#testPrefetchListUponBlockEviction Thread=22 (was 21) Potentially hanging thread: hfile-prefetch-1733957782438 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=290 (was 289) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=338 (was 359), ProcessCount=11 (was 11), AvailableMemoryMB=9546 (was 9550) 2024-12-11T22:56:24,508 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestBucketCachePersister#testPrefetchPersistenceCrash Thread=22, OpenFileDescriptor=290, MaxFileDescriptor=1048576, SystemLoadAverage=338, ProcessCount=11, AvailableMemoryMB=9545 2024-12-11T22:56:24,508 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T22:56:24,511 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/testPrefetchPersistenceCrash 2024-12-11T22:56:24,511 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-11T22:56:24,511 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-11T22:56:24,511 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 3000 2024-12-11T22:56:24,511 INFO [Thread-27 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-11T22:56:24,511 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/testPrefetchPersistenceCrash, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-11T22:56:24,511 WARN [Thread-27 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-11T22:56:24,512 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T22:56:24,512 INFO [Thread-27 {}] bucket.BucketAllocator(343): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-11T22:56:24,512 INFO [Thread-27 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/bucket.persistence is complete. 2024-12-11T22:56:24,519 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 822a282fbd4d45858ea9595f2cd90aad 2024-12-11T22:56:24,527 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch0/822a282fbd4d45858ea9595f2cd90aad is 233, key is aaaaaaaaaaaaaaaaaaaaaabaaababaab/O:1XPgspdzNlntZQ/3075292714023380184/Put/seqid=0 2024-12-11T22:56:24,534 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 3ee468a006a44e3fafcc4080a899faa1 2024-12-11T22:56:24,540 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch1/3ee468a006a44e3fafcc4080a899faa1 is 234, key is aaaaaaaaaaaaaaaaaaaaaaabaabbaaba/Cmt:EoMyse/-7718363739950701151/Put/seqid=0 2024-12-11T22:56:24,541 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch0/822a282fbd4d45858ea9595f2cd90aad, delay=942 ms 2024-12-11T22:56:25,486 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_0. 2024-12-11T22:56:25,487 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_2086. 2024-12-11T22:56:25,489 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_4258. 2024-12-11T22:56:25,489 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_6432. 2024-12-11T22:56:25,490 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_8596. 2024-12-11T22:56:25,491 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_10747. 2024-12-11T22:56:25,492 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_12894. 2024-12-11T22:56:25,492 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_15057. 2024-12-11T22:56:25,493 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_17188. 2024-12-11T22:56:25,493 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_19387. 2024-12-11T22:56:25,493 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_21501. 2024-12-11T22:56:25,494 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_23589. 2024-12-11T22:56:25,494 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_25747. 2024-12-11T22:56:25,494 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_27876. 2024-12-11T22:56:25,495 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_30084. 2024-12-11T22:56:25,495 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_32231. 2024-12-11T22:56:25,495 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_34439. 2024-12-11T22:56:25,496 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_36568. 2024-12-11T22:56:25,496 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_38702. 2024-12-11T22:56:25,496 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_40900. 2024-12-11T22:56:25,496 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_43104. 2024-12-11T22:56:25,497 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_45215. 2024-12-11T22:56:25,497 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_47421. 2024-12-11T22:56:25,497 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_49575. 2024-12-11T22:56:25,497 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_51735. 2024-12-11T22:56:25,498 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_53858. 2024-12-11T22:56:25,498 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_55958. 2024-12-11T22:56:25,498 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_58139. 2024-12-11T22:56:25,498 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_60304. 2024-12-11T22:56:25,498 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_62423. 2024-12-11T22:56:25,498 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_64544. 2024-12-11T22:56:25,499 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_66759. 2024-12-11T22:56:25,499 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_68879. 2024-12-11T22:56:25,499 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_71029. 2024-12-11T22:56:25,499 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_73196. 2024-12-11T22:56:25,499 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_75391. 2024-12-11T22:56:25,499 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_77583. 2024-12-11T22:56:25,500 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_79746. 2024-12-11T22:56:25,500 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_81860. 2024-12-11T22:56:25,500 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_84069. 2024-12-11T22:56:25,500 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_86178. 2024-12-11T22:56:25,500 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_88305. 2024-12-11T22:56:25,501 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_90422. 2024-12-11T22:56:25,501 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_92599. 2024-12-11T22:56:25,501 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_94768. 2024-12-11T22:56:25,501 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_96891. 2024-12-11T22:56:25,501 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_99055. 2024-12-11T22:56:25,501 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_101287. 2024-12-11T22:56:25,502 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_103461. 2024-12-11T22:56:25,502 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_105646. 2024-12-11T22:56:25,502 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_107795. 2024-12-11T22:56:25,502 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_110027. 2024-12-11T22:56:25,502 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_112163. 2024-12-11T22:56:25,502 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_114309. 2024-12-11T22:56:25,503 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_116461. 2024-12-11T22:56:25,503 DEBUG [hfile-prefetch-1733957778986 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 822a282fbd4d45858ea9595f2cd90aad_118443. 2024-12-11T22:56:25,503 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch0/822a282fbd4d45858ea9595f2cd90aad, with total blocks 56, and data blocks 55 2024-12-11T22:56:25,503 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-11T22:56:25,503 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch0/822a282fbd4d45858ea9595f2cd90aad, but only 0 data blocks were cached. Total data blocks for file: 55. Checking for blocks pending cache in cache writer queue. 2024-12-11T22:56:25,503 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch0/822a282fbd4d45858ea9595f2cd90aad. Will sleep 100ms and try the verification again. 2024-12-11T22:56:25,604 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch0/822a282fbd4d45858ea9595f2cd90aad, with total blocks 56, and data blocks 55 2024-12-11T22:56:25,604 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-11T22:56:25,604 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch0/822a282fbd4d45858ea9595f2cd90aad, but only 0 data blocks were cached. Total data blocks for file: 55. Checking for blocks pending cache in cache writer queue. 2024-12-11T22:56:25,604 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch0/822a282fbd4d45858ea9595f2cd90aad. Will sleep 100ms and try the verification again. 2024-12-11T22:56:25,705 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch0/822a282fbd4d45858ea9595f2cd90aad, with total blocks 56, and data blocks 55 2024-12-11T22:56:25,705 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2285): iterating over 3 entries in the backing map 2024-12-11T22:56:25,705 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 0 2024-12-11T22:56:25,705 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 2086 2024-12-11T22:56:25,706 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 32231 2024-12-11T22:56:25,706 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch0/822a282fbd4d45858ea9595f2cd90aad, but only 3 data blocks were cached. Total data blocks for file: 55. Checking for blocks pending cache in cache writer queue. 2024-12-11T22:56:25,706 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch0/822a282fbd4d45858ea9595f2cd90aad. Will sleep 100ms and try the verification again. 2024-12-11T22:56:25,806 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch0/822a282fbd4d45858ea9595f2cd90aad, with total blocks 56, and data blocks 55 2024-12-11T22:56:25,806 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2285): iterating over 56 entries in the backing map 2024-12-11T22:56:25,807 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 0 2024-12-11T22:56:25,807 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 2086 2024-12-11T22:56:25,807 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 4258 2024-12-11T22:56:25,807 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 6432 2024-12-11T22:56:25,807 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 8596 2024-12-11T22:56:25,807 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 10747 2024-12-11T22:56:25,807 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 12894 2024-12-11T22:56:25,807 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 15057 2024-12-11T22:56:25,807 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 17188 2024-12-11T22:56:25,807 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 19387 2024-12-11T22:56:25,807 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 21501 2024-12-11T22:56:25,808 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 23589 2024-12-11T22:56:25,808 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 25747 2024-12-11T22:56:25,808 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 27876 2024-12-11T22:56:25,808 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 30084 2024-12-11T22:56:25,808 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 32231 2024-12-11T22:56:25,808 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 34439 2024-12-11T22:56:25,808 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 36568 2024-12-11T22:56:25,808 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 38702 2024-12-11T22:56:25,808 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 40900 2024-12-11T22:56:25,809 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 43104 2024-12-11T22:56:25,809 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 45215 2024-12-11T22:56:25,809 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 47421 2024-12-11T22:56:25,809 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 49575 2024-12-11T22:56:25,809 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 51735 2024-12-11T22:56:25,809 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 53858 2024-12-11T22:56:25,810 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 55958 2024-12-11T22:56:25,810 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 58139 2024-12-11T22:56:25,810 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 60304 2024-12-11T22:56:25,810 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 62423 2024-12-11T22:56:25,810 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 64544 2024-12-11T22:56:25,810 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 66759 2024-12-11T22:56:25,811 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 68879 2024-12-11T22:56:25,811 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 71029 2024-12-11T22:56:25,811 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 73196 2024-12-11T22:56:25,811 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 75391 2024-12-11T22:56:25,811 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 77583 2024-12-11T22:56:25,811 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 79746 2024-12-11T22:56:25,812 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 81860 2024-12-11T22:56:25,812 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 84069 2024-12-11T22:56:25,812 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 86178 2024-12-11T22:56:25,812 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 88305 2024-12-11T22:56:25,812 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 90422 2024-12-11T22:56:25,813 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 92599 2024-12-11T22:56:25,813 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 94768 2024-12-11T22:56:25,813 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 96891 2024-12-11T22:56:25,813 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 99055 2024-12-11T22:56:25,813 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 101287 2024-12-11T22:56:25,813 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 103461 2024-12-11T22:56:25,814 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 105646 2024-12-11T22:56:25,814 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 107795 2024-12-11T22:56:25,814 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 110027 2024-12-11T22:56:25,814 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 112163 2024-12-11T22:56:25,814 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 114309 2024-12-11T22:56:25,815 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 116461 2024-12-11T22:56:25,815 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2293): found block for file 822a282fbd4d45858ea9595f2cd90aad in the backing map. Acquiring read lock for offset 118443 2024-12-11T22:56:25,815 DEBUG [hfile-prefetch-1733957778986 {}] bucket.BucketCache(2305): File /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch0/822a282fbd4d45858ea9595f2cd90aad has now been fully cached. 2024-12-11T22:56:25,815 DEBUG [hfile-prefetch-1733957778986 {}] hfile.PrefetchExecutor(121): Prefetch completed for 822a282fbd4d45858ea9595f2cd90aad 2024-12-11T22:56:26,547 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch1/3ee468a006a44e3fafcc4080a899faa1, delay=956 ms 2024-12-11T22:56:27,507 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_0. 2024-12-11T22:56:27,509 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_2111. 2024-12-11T22:56:27,510 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_4265. 2024-12-11T22:56:27,511 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_6443. 2024-12-11T22:56:27,512 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-11T22:56:27,512 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-11T22:56:27,512 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_8530. 2024-12-11T22:56:27,512 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-11T22:56:27,512 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 56, fullycachedFiles size: 1, chunkSize: 10000 2024-12-11T22:56:27,513 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_10653. 2024-12-11T22:56:27,513 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_12790. 2024-12-11T22:56:27,513 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_14898. 2024-12-11T22:56:27,514 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_16992. 2024-12-11T22:56:27,514 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_19186. 2024-12-11T22:56:27,514 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_21306. 2024-12-11T22:56:27,514 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_23479. 2024-12-11T22:56:27,515 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_25628. 2024-12-11T22:56:27,515 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_27742. 2024-12-11T22:56:27,515 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_29915. 2024-12-11T22:56:27,516 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_32040. 2024-12-11T22:56:27,516 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_34264. 2024-12-11T22:56:27,516 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_36364. 2024-12-11T22:56:27,517 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_38492. 2024-12-11T22:56:27,517 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_40618. 2024-12-11T22:56:27,517 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_42715. 2024-12-11T22:56:27,517 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_44871. 2024-12-11T22:56:27,517 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_46989. 2024-12-11T22:56:27,517 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 56, fullycachedFiles size: 1 2024-12-11T22:56:27,517 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 56, fullycachedFiles size: 1, file name: bucket.persistence1733957787512 2024-12-11T22:56:27,517 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_49153. 2024-12-11T22:56:27,518 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-11T22:56:27,518 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_51251. 2024-12-11T22:56:27,518 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_53419. 2024-12-11T22:56:27,518 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_55532. 2024-12-11T22:56:27,518 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_57660. 2024-12-11T22:56:27,518 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_59765. 2024-12-11T22:56:27,518 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_61930. 2024-12-11T22:56:27,519 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_64156. 2024-12-11T22:56:27,519 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_66264. 2024-12-11T22:56:27,519 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_68445. 2024-12-11T22:56:27,519 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_70552. 2024-12-11T22:56:27,519 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_72773. 2024-12-11T22:56:27,519 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_74890. 2024-12-11T22:56:27,520 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_76976. 2024-12-11T22:56:27,520 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_79129. 2024-12-11T22:56:27,520 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_81332. 2024-12-11T22:56:27,520 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_83535. 2024-12-11T22:56:27,520 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_85663. 2024-12-11T22:56:27,520 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_87803. 2024-12-11T22:56:27,521 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_89899. 2024-12-11T22:56:27,521 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_92000. 2024-12-11T22:56:27,521 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_94158. 2024-12-11T22:56:27,521 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_96298. 2024-12-11T22:56:27,521 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_98419. 2024-12-11T22:56:27,521 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_100576. 2024-12-11T22:56:27,522 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_102758. 2024-12-11T22:56:27,522 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_104978. 2024-12-11T22:56:27,522 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_107178. 2024-12-11T22:56:27,522 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_109408. 2024-12-11T22:56:27,522 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_111636. 2024-12-11T22:56:27,523 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_113791. 2024-12-11T22:56:27,523 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_115994. 2024-12-11T22:56:27,523 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_118121. 2024-12-11T22:56:27,523 DEBUG [hfile-prefetch-1733957782438 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3ee468a006a44e3fafcc4080a899faa1_118852. 2024-12-11T22:56:27,523 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch1/3ee468a006a44e3fafcc4080a899faa1, with total blocks 57, and data blocks 56 2024-12-11T22:56:27,523 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2285): iterating over 56 entries in the backing map 2024-12-11T22:56:27,523 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch1/3ee468a006a44e3fafcc4080a899faa1, but only 0 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue. 2024-12-11T22:56:27,523 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch1/3ee468a006a44e3fafcc4080a899faa1. Will sleep 100ms and try the verification again. 2024-12-11T22:56:27,624 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch1/3ee468a006a44e3fafcc4080a899faa1, with total blocks 57, and data blocks 56 2024-12-11T22:56:27,624 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2285): iterating over 56 entries in the backing map 2024-12-11T22:56:27,624 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch1/3ee468a006a44e3fafcc4080a899faa1, but only 0 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue. 2024-12-11T22:56:27,624 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch1/3ee468a006a44e3fafcc4080a899faa1. Will sleep 100ms and try the verification again. 2024-12-11T22:56:27,725 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch1/3ee468a006a44e3fafcc4080a899faa1, with total blocks 57, and data blocks 56 2024-12-11T22:56:27,725 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2285): iterating over 59 entries in the backing map 2024-12-11T22:56:27,725 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 0 2024-12-11T22:56:27,725 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 2111 2024-12-11T22:56:27,726 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 8530 2024-12-11T22:56:27,726 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2308): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch1/3ee468a006a44e3fafcc4080a899faa1, but only 3 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue. 2024-12-11T22:56:27,726 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2318): There are still blocks pending caching for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch1/3ee468a006a44e3fafcc4080a899faa1. Will sleep 100ms and try the verification again. 2024-12-11T22:56:27,826 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2281): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch1/3ee468a006a44e3fafcc4080a899faa1, with total blocks 57, and data blocks 56 2024-12-11T22:56:27,826 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2285): iterating over 113 entries in the backing map 2024-12-11T22:56:27,827 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 0 2024-12-11T22:56:27,827 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 2111 2024-12-11T22:56:27,827 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 4265 2024-12-11T22:56:27,827 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 6443 2024-12-11T22:56:27,827 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 8530 2024-12-11T22:56:27,827 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 10653 2024-12-11T22:56:27,827 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 12790 2024-12-11T22:56:27,827 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 14898 2024-12-11T22:56:27,827 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 16992 2024-12-11T22:56:27,827 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 19186 2024-12-11T22:56:27,828 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 21306 2024-12-11T22:56:27,828 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 23479 2024-12-11T22:56:27,828 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 25628 2024-12-11T22:56:27,828 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 27742 2024-12-11T22:56:27,828 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 29915 2024-12-11T22:56:27,828 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 32040 2024-12-11T22:56:27,828 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 34264 2024-12-11T22:56:27,828 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 36364 2024-12-11T22:56:27,828 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 38492 2024-12-11T22:56:27,828 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 40618 2024-12-11T22:56:27,829 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 42715 2024-12-11T22:56:27,829 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 44871 2024-12-11T22:56:27,829 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 46989 2024-12-11T22:56:27,829 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 49153 2024-12-11T22:56:27,829 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 51251 2024-12-11T22:56:27,829 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 53419 2024-12-11T22:56:27,829 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 55532 2024-12-11T22:56:27,829 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 57660 2024-12-11T22:56:27,829 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 59765 2024-12-11T22:56:27,830 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 61930 2024-12-11T22:56:27,830 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 64156 2024-12-11T22:56:27,830 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 66264 2024-12-11T22:56:27,830 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 68445 2024-12-11T22:56:27,830 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 70552 2024-12-11T22:56:27,830 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 72773 2024-12-11T22:56:27,830 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 74890 2024-12-11T22:56:27,830 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 76976 2024-12-11T22:56:27,831 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 79129 2024-12-11T22:56:27,831 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 81332 2024-12-11T22:56:27,831 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 83535 2024-12-11T22:56:27,831 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 85663 2024-12-11T22:56:27,831 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 87803 2024-12-11T22:56:27,831 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 89899 2024-12-11T22:56:27,831 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 92000 2024-12-11T22:56:27,831 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 94158 2024-12-11T22:56:27,831 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 96298 2024-12-11T22:56:27,831 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 98419 2024-12-11T22:56:27,832 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 100576 2024-12-11T22:56:27,832 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 102758 2024-12-11T22:56:27,832 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 104978 2024-12-11T22:56:27,832 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 107178 2024-12-11T22:56:27,832 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 109408 2024-12-11T22:56:27,832 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 111636 2024-12-11T22:56:27,832 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 113791 2024-12-11T22:56:27,832 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 115994 2024-12-11T22:56:27,832 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 118121 2024-12-11T22:56:27,832 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2293): found block for file 3ee468a006a44e3fafcc4080a899faa1 in the backing map. Acquiring read lock for offset 118852 2024-12-11T22:56:27,833 DEBUG [hfile-prefetch-1733957782438 {}] bucket.BucketCache(2305): File /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/TestPrefetch1/3ee468a006a44e3fafcc4080a899faa1 has now been fully cached. 2024-12-11T22:56:27,833 DEBUG [hfile-prefetch-1733957782438 {}] hfile.PrefetchExecutor(121): Prefetch completed for 3ee468a006a44e3fafcc4080a899faa1 2024-12-11T22:56:30,518 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-11T22:56:30,518 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-11T22:56:30,519 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-11T22:56:30,519 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 113, fullycachedFiles size: 2, chunkSize: 10000 2024-12-11T22:56:30,528 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 113, fullycachedFiles size: 2 2024-12-11T22:56:30,528 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 113, fullycachedFiles size: 2, file name: bucket.persistence1733957790518 2024-12-11T22:56:30,528 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-11T22:56:31,554 INFO [Time-limited test {}] bucket.BucketCache(1738): Disabling cache 2024-12-11T22:56:31,555 INFO [Time-limited test {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4f9723e8-69bc-fede-3ddb-eb4d59f53f1d/bucket.persistence 2024-12-11T22:56:31,555 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-11T22:56:31,555 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-11T22:56:31,555 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-11T22:56:31,555 INFO [Time-limited test {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-11T22:56:31,556 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-11T22:56:31,566 DEBUG [Time-limited test {}] bucket.BucketCache(1487): Thread Time-limited test started persisting bucket cache to file 2024-12-11T22:56:31,566 DEBUG [Time-limited test {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-11T22:56:31,566 DEBUG [Time-limited test {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 113, fullycachedFiles size: 2, chunkSize: 10000 2024-12-11T22:56:31,576 DEBUG [Time-limited test {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 113, fullycachedFiles size: 2 2024-12-11T22:56:31,576 DEBUG [Time-limited test {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 113, fullycachedFiles size: 2, file name: bucket.persistence1733957791566 2024-12-11T22:56:31,577 DEBUG [Time-limited test {}] bucket.BucketCache(1509): Thread Time-limited test finished persisting bucket cache to file, renaming 2024-12-11T22:56:31,577 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T22:56:31,578 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T22:56:31,583 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestBucketCachePersister#testPrefetchPersistenceCrash Thread=23 (was 22) Potentially hanging thread: hfile-prefetch-1733957784541 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=292 (was 290) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=311 (was 338), ProcessCount=11 (was 11), AvailableMemoryMB=9535 (was 9545)