2024-04-14 11:12:52,571 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@f79e 2024-04-14 11:12:52,587 main DEBUG Took 0.013245 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-04-14 11:12:52,588 main DEBUG PluginManager 'Core' found 129 plugins 2024-04-14 11:12:52,588 main DEBUG PluginManager 'Level' found 0 plugins 2024-04-14 11:12:52,590 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-04-14 11:12:52,591 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-14 11:12:52,602 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-04-14 11:12:52,618 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-14 11:12:52,620 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-14 11:12:52,621 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-14 11:12:52,621 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-14 11:12:52,621 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-14 11:12:52,622 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-14 11:12:52,623 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-14 11:12:52,623 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-14 11:12:52,624 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-14 11:12:52,624 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-14 11:12:52,625 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-14 11:12:52,625 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-14 11:12:52,626 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-14 11:12:52,626 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-14 11:12:52,627 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-14 11:12:52,627 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-14 11:12:52,628 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-14 11:12:52,628 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-14 11:12:52,629 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-14 11:12:52,629 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-14 11:12:52,630 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-14 11:12:52,630 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-14 11:12:52,631 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-14 11:12:52,631 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-14 11:12:52,631 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-14 11:12:52,632 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-04-14 11:12:52,634 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-14 11:12:52,637 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-04-14 11:12:52,639 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-04-14 11:12:52,640 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-04-14 11:12:52,641 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-04-14 11:12:52,641 main DEBUG PluginManager 'Converter' found 47 plugins 2024-04-14 11:12:52,654 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-04-14 11:12:52,658 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-04-14 11:12:52,659 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-04-14 11:12:52,659 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-04-14 11:12:52,660 main DEBUG createAppenders(={Console}) 2024-04-14 11:12:52,661 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@f79e initialized 2024-04-14 11:12:52,661 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@f79e 2024-04-14 11:12:52,661 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@f79e OK. 2024-04-14 11:12:52,662 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-04-14 11:12:52,662 main DEBUG OutputStream closed 2024-04-14 11:12:52,662 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-04-14 11:12:52,663 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-04-14 11:12:52,663 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@6aeb35e6 OK 2024-04-14 11:12:52,719 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586 2024-04-14 11:12:52,722 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=StatusLogger 2024-04-14 11:12:52,723 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=ContextSelector 2024-04-14 11:12:52,725 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name= 2024-04-14 11:12:52,726 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.directory 2024-04-14 11:12:52,726 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-04-14 11:12:52,727 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.zookeeper 2024-04-14 11:12:52,727 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-04-14 11:12:52,728 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-04-14 11:12:52,728 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-04-14 11:12:52,728 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase 2024-04-14 11:12:52,729 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop 2024-04-14 11:12:52,729 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-04-14 11:12:52,730 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-04-14 11:12:52,730 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-04-14 11:12:52,731 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-04-14 11:12:52,731 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-04-14 11:12:52,732 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Appenders,name=Console 2024-04-14 11:12:52,736 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2024-04-14 11:12:52,741 main DEBUG Reconfiguration complete for context[name=1b6d3586] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.7.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@23c30a20) with optional ClassLoader: null 2024-04-14 11:12:52,741 main DEBUG Shutdown hook enabled. Registering a new one. 2024-04-14 11:12:52,742 main DEBUG LoggerContext[name=1b6d3586, org.apache.logging.log4j.core.LoggerContext@23c30a20] started OK. 2024-04-14T11:12:52,762 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.bucket.TestBucketCachePersister timeout: 13 mins 2024-04-14 11:12:52,766 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-04-14 11:12:52,767 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2024-04-14T11:12:53,082 DEBUG [main {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc221547-0085-fa7f-3168-33a7e0b8cd32 2024-04-14T11:12:53,112 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestBucketCachePersister#testPrefetchBlockEvictionWhilePrefetchRunning Thread=10, OpenFileDescriptor=306, MaxFileDescriptor=60000, SystemLoadAverage=224, ProcessCount=170, AvailableMemoryMB=3358 2024-04-14T11:12:53,251 WARN [Time-limited test {}] util.NativeCodeLoader(62): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-04-14T11:12:53,736 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-04-14T11:12:53,766 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc221547-0085-fa7f-3168-33a7e0b8cd32/testPrefetchBlockEvictionWhilePrefetchRunning 2024-04-14T11:12:53,767 INFO [Time-limited test {}] bucket.BucketCache(305): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2024-04-14T11:12:53,768 INFO [Time-limited test {}] bucket.BucketCache(393): Starting BucketCachePersisterThread 2024-04-14T11:12:53,769 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 200 2024-04-14T11:12:53,770 INFO [Time-limited test {}] bucket.BucketCache(1344): Started retrieving bucket cache from file 2024-04-14T11:12:53,770 WARN [Time-limited test {}] bucket.BucketCache(1347): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-04-14T11:12:53,782 INFO [Time-limited test {}] bucket.BucketAllocator(339): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-04-14T11:12:53,784 INFO [Time-limited test {}] bucket.BucketCache(356): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc221547-0085-fa7f-3168-33a7e0b8cd32/testPrefetchBlockEvictionWhilePrefetchRunning, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc221547-0085-fa7f-3168-33a7e0b8cd32/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-04-14T11:12:53,787 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-04-14T11:12:53,808 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-04-14T11:12:53,862 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8e7b3b498f924d7d8d52c2f6ffea19a5 2024-04-14T11:12:54,029 WARN [Time-limited test {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-04-14T11:12:54,072 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-04-14T11:12:54,119 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(813): Len of the biggest cell in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc221547-0085-fa7f-3168-33a7e0b8cd32/TestPrefetch3/8e7b3b498f924d7d8d52c2f6ffea19a5 is 234, key is aaaaaaaaaaaaaaaaaaaaaaaabbababaa//3678165378028042832/DeleteColumn/seqid=0 2024-04-14T11:12:54,279 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(92): Prefetch requested for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc221547-0085-fa7f-3168-33a7e0b8cd32/TestPrefetch3/8e7b3b498f924d7d8d52c2f6ffea19a5, delay=963 ms 2024-04-14T11:12:55,256 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_0. 2024-04-14T11:12:55,337 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_2124. 2024-04-14T11:12:55,338 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_4321. 2024-04-14T11:12:55,339 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_6453. 2024-04-14T11:12:55,339 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_8632. 2024-04-14T11:12:55,340 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_10792. 2024-04-14T11:12:55,340 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_12910. 2024-04-14T11:12:55,341 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_15175. 2024-04-14T11:12:55,341 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_17354. 2024-04-14T11:12:55,342 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_19488. 2024-04-14T11:12:55,342 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_21678. 2024-04-14T11:12:55,342 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_23786. 2024-04-14T11:12:55,343 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_25878. 2024-04-14T11:12:55,343 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_28003. 2024-04-14T11:12:55,344 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_30154. 2024-04-14T11:12:55,344 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_32348. 2024-04-14T11:12:55,345 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_34503. 2024-04-14T11:12:55,345 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_36704. 2024-04-14T11:12:55,345 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_38815. 2024-04-14T11:12:55,346 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_40903. 2024-04-14T11:12:55,346 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_43125. 2024-04-14T11:12:55,347 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_45233. 2024-04-14T11:12:55,347 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_47405. 2024-04-14T11:12:55,347 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_49587. 2024-04-14T11:12:55,348 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_51765. 2024-04-14T11:12:55,348 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_53885. 2024-04-14T11:12:55,349 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_56040. 2024-04-14T11:12:55,349 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_58197. 2024-04-14T11:12:55,349 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_60291. 2024-04-14T11:12:55,350 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_62468. 2024-04-14T11:12:55,350 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_64555. 2024-04-14T11:12:55,350 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_66642. 2024-04-14T11:12:55,351 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_68873. 2024-04-14T11:12:55,351 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_71048. 2024-04-14T11:12:55,351 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_73149. 2024-04-14T11:12:55,352 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_75247. 2024-04-14T11:12:55,352 DEBUG [Time-limited test {}] bucket.BucketCache(739): removed key 8e7b3b498f924d7d8d52c2f6ffea19a5_17354 from back map with offset lock 519168 in the evict process 2024-04-14T11:12:55,352 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_77416. 2024-04-14T11:12:55,353 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_79517. 2024-04-14T11:12:55,353 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_81742. 2024-04-14T11:12:55,353 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_83934. 2024-04-14T11:12:55,354 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_86157. 2024-04-14T11:12:55,354 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_88250. 2024-04-14T11:12:55,355 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_90357. 2024-04-14T11:12:55,355 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_92501. 2024-04-14T11:12:55,355 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_94723. 2024-04-14T11:12:55,355 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_96842. 2024-04-14T11:12:55,356 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_98991. 2024-04-14T11:12:55,356 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_101132. 2024-04-14T11:12:55,356 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_103337. 2024-04-14T11:12:55,357 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_105556. 2024-04-14T11:12:55,357 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_107702. 2024-04-14T11:12:55,358 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_109855. 2024-04-14T11:12:55,358 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_111981. 2024-04-14T11:12:55,358 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_114099. 2024-04-14T11:12:55,359 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_116263. 2024-04-14T11:12:55,359 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_118442. 2024-04-14T11:12:55,360 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 8e7b3b498f924d7d8d52c2f6ffea19a5_119065. 2024-04-14T11:12:55,360 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2064): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc221547-0085-fa7f-3168-33a7e0b8cd32/TestPrefetch3/8e7b3b498f924d7d8d52c2f6ffea19a5, with total blocks 57, and data blocks 56 2024-04-14T11:12:55,361 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2068): iterating over 27 entries in the backing map 2024-04-14T11:12:55,363 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_36704 in the backing map. Acquiring read lock for offset 473088 2024-04-14T11:12:55,363 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_0 in the backing map. Acquiring read lock for offset 525312 2024-04-14T11:12:55,363 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_40903 in the backing map. Acquiring read lock for offset 463872 2024-04-14T11:12:55,363 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_32348 in the backing map. Acquiring read lock for offset 476160 2024-04-14T11:12:55,363 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_28003 in the backing map. Acquiring read lock for offset 494592 2024-04-14T11:12:55,364 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_64555 in the backing map. Acquiring read lock for offset 457728 2024-04-14T11:12:55,364 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_19488 in the backing map. Acquiring read lock for offset 500736 2024-04-14T11:12:55,364 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_23786 in the backing map. Acquiring read lock for offset 479232 2024-04-14T11:12:55,364 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_68873 in the backing map. Acquiring read lock for offset 436224 2024-04-14T11:12:55,364 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_60291 in the backing map. Acquiring read lock for offset 427008 2024-04-14T11:12:55,364 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_15175 in the backing map. Acquiring read lock for offset 497664 2024-04-14T11:12:55,365 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_10792 in the backing map. Acquiring read lock for offset 509952 2024-04-14T11:12:55,365 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_51765 in the backing map. Acquiring read lock for offset 460800 2024-04-14T11:12:55,365 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_96842 in the backing map. Acquiring read lock for offset 399360 2024-04-14T11:12:55,365 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_56040 in the backing map. Acquiring read lock for offset 454656 2024-04-14T11:12:55,365 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_101132 in the backing map. Acquiring read lock for offset 396288 2024-04-14T11:12:55,366 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_92501 in the backing map. Acquiring read lock for offset 405504 2024-04-14T11:12:55,366 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_47405 in the backing map. Acquiring read lock for offset 445440 2024-04-14T11:12:55,366 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_6453 in the backing map. Acquiring read lock for offset 506880 2024-04-14T11:12:55,366 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_43125 in the backing map. Acquiring read lock for offset 470016 2024-04-14T11:12:55,366 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_2124 in the backing map. Acquiring read lock for offset 513024 2024-04-14T11:12:55,366 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_88250 in the backing map. Acquiring read lock for offset 411648 2024-04-14T11:12:55,367 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_38815 in the backing map. Acquiring read lock for offset 466944 2024-04-14T11:12:55,367 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_79517 in the backing map. Acquiring read lock for offset 430080 2024-04-14T11:12:55,367 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_34503 in the backing map. Acquiring read lock for offset 482304 2024-04-14T11:12:55,368 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_71048 in the backing map. Acquiring read lock for offset 433152 2024-04-14T11:12:55,368 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_25878 in the backing map. Acquiring read lock for offset 485376 2024-04-14T11:12:55,368 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_75247 in the backing map. Acquiring read lock for offset 420864 2024-04-14T11:12:55,368 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_30154 in the backing map. Acquiring read lock for offset 488448 2024-04-14T11:12:55,369 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_66642 in the backing map. Acquiring read lock for offset 423936 2024-04-14T11:12:55,369 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_21678 in the backing map. Acquiring read lock for offset 491520 2024-04-14T11:12:55,369 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_107702 in the backing map. Acquiring read lock for offset 390144 2024-04-14T11:12:55,369 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_58197 in the backing map. Acquiring read lock for offset 451584 2024-04-14T11:12:55,369 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_62468 in the backing map. Acquiring read lock for offset 439296 2024-04-14T11:12:55,370 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_103337 in the backing map. Acquiring read lock for offset 393216 2024-04-14T11:12:55,370 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_12910 in the backing map. Acquiring read lock for offset 503808 2024-04-14T11:12:55,370 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_53885 in the backing map. Acquiring read lock for offset 442368 2024-04-14T11:12:55,370 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_94723 in the backing map. Acquiring read lock for offset 408576 2024-04-14T11:12:55,370 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_8632 in the backing map. Acquiring read lock for offset 516096 2024-04-14T11:12:55,370 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_49587 in the backing map. Acquiring read lock for offset 519168 2024-04-14T11:12:55,371 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-04-14T11:12:55,371 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_4321 in the backing map. Acquiring read lock for offset 522240 2024-04-14T11:12:55,371 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1310): Thread bucket-cache-persister started persisting bucket cache to file 2024-04-14T11:12:55,371 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_45233 in the backing map. Acquiring read lock for offset 448512 2024-04-14T11:12:55,371 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2096): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc221547-0085-fa7f-3168-33a7e0b8cd32/TestPrefetch3/8e7b3b498f924d7d8d52c2f6ffea19a5, but only 42 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue. 2024-04-14T11:12:55,373 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2105): There are still blocks pending caching for file 8e7b3b498f924d7d8d52c2f6ffea19a5. Will sleep 100ms and try the verification again. 2024-04-14T11:12:55,474 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2064): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc221547-0085-fa7f-3168-33a7e0b8cd32/TestPrefetch3/8e7b3b498f924d7d8d52c2f6ffea19a5, with total blocks 57, and data blocks 56 2024-04-14T11:12:55,474 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2068): iterating over 56 entries in the backing map 2024-04-14T11:12:55,474 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_36704 in the backing map. Acquiring read lock for offset 473088 2024-04-14T11:12:55,475 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_81742 in the backing map. Acquiring read lock for offset 414720 2024-04-14T11:12:55,475 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_0 in the backing map. Acquiring read lock for offset 525312 2024-04-14T11:12:55,475 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_40903 in the backing map. Acquiring read lock for offset 463872 2024-04-14T11:12:55,475 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_32348 in the backing map. Acquiring read lock for offset 476160 2024-04-14T11:12:55,475 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_77416 in the backing map. Acquiring read lock for offset 417792 2024-04-14T11:12:55,475 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_118442 in the backing map. Acquiring read lock for offset 356352 2024-04-14T11:12:55,476 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_28003 in the backing map. Acquiring read lock for offset 494592 2024-04-14T11:12:55,476 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_109855 in the backing map. Acquiring read lock for offset 365568 2024-04-14T11:12:55,476 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_73149 in the backing map. Acquiring read lock for offset 387072 2024-04-14T11:12:55,476 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_114099 in the backing map. Acquiring read lock for offset 368640 2024-04-14T11:12:55,476 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_105556 in the backing map. Acquiring read lock for offset 402432 2024-04-14T11:12:55,476 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_64555 in the backing map. Acquiring read lock for offset 457728 2024-04-14T11:12:55,476 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_19488 in the backing map. Acquiring read lock for offset 500736 2024-04-14T11:12:55,477 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_23786 in the backing map. Acquiring read lock for offset 479232 2024-04-14T11:12:55,477 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_68873 in the backing map. Acquiring read lock for offset 436224 2024-04-14T11:12:55,477 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_60291 in the backing map. Acquiring read lock for offset 427008 2024-04-14T11:12:55,477 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_15175 in the backing map. Acquiring read lock for offset 497664 2024-04-14T11:12:55,477 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_10792 in the backing map. Acquiring read lock for offset 509952 2024-04-14T11:12:55,477 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_51765 in the backing map. Acquiring read lock for offset 460800 2024-04-14T11:12:55,477 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_96842 in the backing map. Acquiring read lock for offset 399360 2024-04-14T11:12:55,477 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_56040 in the backing map. Acquiring read lock for offset 454656 2024-04-14T11:12:55,478 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_101132 in the backing map. Acquiring read lock for offset 396288 2024-04-14T11:12:55,478 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_92501 in the backing map. Acquiring read lock for offset 405504 2024-04-14T11:12:55,478 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_47405 in the backing map. Acquiring read lock for offset 445440 2024-04-14T11:12:55,478 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_6453 in the backing map. Acquiring read lock for offset 506880 2024-04-14T11:12:55,478 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_43125 in the backing map. Acquiring read lock for offset 470016 2024-04-14T11:12:55,478 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_2124 in the backing map. Acquiring read lock for offset 513024 2024-04-14T11:12:55,479 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_88250 in the backing map. Acquiring read lock for offset 411648 2024-04-14T11:12:55,479 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_83934 in the backing map. Acquiring read lock for offset 384000 2024-04-14T11:12:55,479 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_38815 in the backing map. Acquiring read lock for offset 466944 2024-04-14T11:12:55,479 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_116263 in the backing map. Acquiring read lock for offset 362496 2024-04-14T11:12:55,479 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_79517 in the backing map. Acquiring read lock for offset 430080 2024-04-14T11:12:55,479 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_34503 in the backing map. Acquiring read lock for offset 482304 2024-04-14T11:12:55,480 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_111981 in the backing map. Acquiring read lock for offset 371712 2024-04-14T11:12:55,480 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_71048 in the backing map. Acquiring read lock for offset 433152 2024-04-14T11:12:55,480 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_25878 in the backing map. Acquiring read lock for offset 485376 2024-04-14T11:12:55,480 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_75247 in the backing map. Acquiring read lock for offset 420864 2024-04-14T11:12:55,480 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_30154 in the backing map. Acquiring read lock for offset 488448 2024-04-14T11:12:55,480 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_66642 in the backing map. Acquiring read lock for offset 423936 2024-04-14T11:12:55,481 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_21678 in the backing map. Acquiring read lock for offset 491520 2024-04-14T11:12:55,481 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_107702 in the backing map. Acquiring read lock for offset 390144 2024-04-14T11:12:55,481 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_58197 in the backing map. Acquiring read lock for offset 451584 2024-04-14T11:12:55,481 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_62468 in the backing map. Acquiring read lock for offset 439296 2024-04-14T11:12:55,481 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_103337 in the backing map. Acquiring read lock for offset 393216 2024-04-14T11:12:55,481 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_12910 in the backing map. Acquiring read lock for offset 503808 2024-04-14T11:12:55,481 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_53885 in the backing map. Acquiring read lock for offset 442368 2024-04-14T11:12:55,481 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_98991 in the backing map. Acquiring read lock for offset 374784 2024-04-14T11:12:55,481 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_94723 in the backing map. Acquiring read lock for offset 408576 2024-04-14T11:12:55,481 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_8632 in the backing map. Acquiring read lock for offset 516096 2024-04-14T11:12:55,481 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_49587 in the backing map. Acquiring read lock for offset 519168 2024-04-14T11:12:55,482 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_86157 in the backing map. Acquiring read lock for offset 380928 2024-04-14T11:12:55,482 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_4321 in the backing map. Acquiring read lock for offset 522240 2024-04-14T11:12:55,482 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_90357 in the backing map. Acquiring read lock for offset 377856 2024-04-14T11:12:55,482 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 8e7b3b498f924d7d8d52c2f6ffea19a5_45233 in the backing map. Acquiring read lock for offset 448512 2024-04-14T11:12:55,482 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2096): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc221547-0085-fa7f-3168-33a7e0b8cd32/TestPrefetch3/8e7b3b498f924d7d8d52c2f6ffea19a5, but only 55 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue. 2024-04-14T11:12:55,483 INFO [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2116): We found only 55 data blocks cached from a total of 56 for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc221547-0085-fa7f-3168-33a7e0b8cd32/TestPrefetch3/8e7b3b498f924d7d8d52c2f6ffea19a5, but no blocks pending caching. Maybe cache is full or evictions happened concurrently to cache prefetch. 2024-04-14T11:12:55,483 WARN [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(151): Prefetch path=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc221547-0085-fa7f-3168-33a7e0b8cd32/TestPrefetch3/8e7b3b498f924d7d8d52c2f6ffea19a5, offset=119358, end=119358 java.lang.IllegalMonitorStateException: attempt to unlock read lock, not locked by current thread at java.util.concurrent.locks.ReentrantReadWriteLock$Sync.unmatchedUnlockException(ReentrantReadWriteLock.java:444) ~[?:1.8.0_362] at java.util.concurrent.locks.ReentrantReadWriteLock$Sync.tryReleaseShared(ReentrantReadWriteLock.java:428) ~[?:1.8.0_362] at java.util.concurrent.locks.AbstractQueuedSynchronizer.releaseShared(AbstractQueuedSynchronizer.java:1341) ~[?:1.8.0_362] at java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.unlock(ReentrantReadWriteLock.java:881) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2125) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:141) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_362] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) ~[?:1.8.0_362] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:12:55,494 DEBUG [hfile-prefetch-1713093174282 {}] hfile.PrefetchExecutor(108): Prefetch completed for 8e7b3b498f924d7d8d52c2f6ffea19a5 2024-04-14T11:12:55,504 INFO [Time-limited test {}] bucket.BucketCache(1534): Disabling cache 2024-04-14T11:12:55,504 INFO [Time-limited test {}] bucket.BucketCache(1558): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc221547-0085-fa7f-3168-33a7e0b8cd32/bucket.persistence 2024-04-14T11:12:55,504 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1094): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-04-14T11:12:55,504 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1094): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-04-14T11:12:55,505 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1094): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-04-14T11:12:55,505 INFO [Time-limited test {}] bucket.BucketCache(1564): Shutting down cache persister thread. 2024-04-14T11:12:55,512 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1323): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-04-14T11:12:55,513 INFO [bucket-cache-persister {}] bucket.BucketCachePersister(62): Finishing cache persister thread. 2024-04-14T11:12:55,515 DEBUG [Time-limited test {}] bucket.BucketCache(1310): Thread Time-limited test started persisting bucket cache to file 2024-04-14T11:12:55,520 DEBUG [Time-limited test {}] bucket.BucketCache(1323): Thread Time-limited test finished persisting bucket cache to file, renaming 2024-04-14T11:12:55,521 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-04-14T11:12:55,522 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-04-14T11:12:55,523 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-04-14T11:12:55,536 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestBucketCachePersister#testPrefetchBlockEvictionWhilePrefetchRunning Thread=14 (was 10) Potentially hanging thread: hfile-prefetch-1713093174282 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Timer for 'HBase' metrics system java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.lang.Object.wait(Native Method) java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:144) java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:165) org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:3712) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: HBase-Metrics2-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=308 (was 306) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=229 (was 224) - SystemLoadAverage LEAK? -, ProcessCount=170 (was 170), AvailableMemoryMB=3196 (was 3358) 2024-04-14T11:12:55,547 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestBucketCachePersister#testPrefetchPersistenceCrashNegative Thread=14, OpenFileDescriptor=308, MaxFileDescriptor=60000, SystemLoadAverage=229, ProcessCount=170, AvailableMemoryMB=3195 2024-04-14T11:12:55,548 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-04-14T11:12:55,549 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc221547-0085-fa7f-3168-33a7e0b8cd32/testPrefetchPersistenceCrashNegative 2024-04-14T11:12:55,549 INFO [Time-limited test {}] bucket.BucketCache(305): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2024-04-14T11:12:55,549 INFO [Time-limited test {}] bucket.BucketCache(393): Starting BucketCachePersisterThread 2024-04-14T11:12:55,549 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 9223372036854775807 2024-04-14T11:12:55,550 INFO [Time-limited test {}] bucket.BucketCache(1344): Started retrieving bucket cache from file 2024-04-14T11:12:55,550 WARN [Time-limited test {}] bucket.BucketCache(1347): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-04-14T11:12:55,550 INFO [Time-limited test {}] bucket.BucketAllocator(339): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-04-14T11:12:55,551 INFO [Time-limited test {}] bucket.BucketCache(356): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc221547-0085-fa7f-3168-33a7e0b8cd32/testPrefetchPersistenceCrashNegative, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc221547-0085-fa7f-3168-33a7e0b8cd32/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-04-14T11:12:55,552 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-04-14T11:12:55,556 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 0968972745424fb6b3a440a03184abc3 2024-04-14T11:12:55,570 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(813): Len of the biggest cell in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc221547-0085-fa7f-3168-33a7e0b8cd32/TestPrefetch2/0968972745424fb6b3a440a03184abc3 is 256, key is aaaaaaaaaaaaaaaaaaaaaabbabbbabbb/vYKSjf0RM:9YM/-182794218876027487/DeleteColumn/seqid=0 2024-04-14T11:12:55,572 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(92): Prefetch requested for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc221547-0085-fa7f-3168-33a7e0b8cd32/TestPrefetch2/0968972745424fb6b3a440a03184abc3, delay=901 ms 2024-04-14T11:12:56,474 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_0. 2024-04-14T11:12:56,475 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_2129. 2024-04-14T11:12:56,476 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_4302. 2024-04-14T11:12:56,476 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_6425. 2024-04-14T11:12:56,476 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_8515. 2024-04-14T11:12:56,477 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_10648. 2024-04-14T11:12:56,477 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_12797. 2024-04-14T11:12:56,477 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_14969. 2024-04-14T11:12:56,477 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_17059. 2024-04-14T11:12:56,478 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_19170. 2024-04-14T11:12:56,478 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_21263. 2024-04-14T11:12:56,478 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_23349. 2024-04-14T11:12:56,478 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_25509. 2024-04-14T11:12:56,478 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_27614. 2024-04-14T11:12:56,479 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_29768. 2024-04-14T11:12:56,479 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_31907. 2024-04-14T11:12:56,479 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_34147. 2024-04-14T11:12:56,480 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_36243. 2024-04-14T11:12:56,480 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_38334. 2024-04-14T11:12:56,480 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_40437. 2024-04-14T11:12:56,480 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_42653. 2024-04-14T11:12:56,481 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_44790. 2024-04-14T11:12:56,481 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_46949. 2024-04-14T11:12:56,481 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_49114. 2024-04-14T11:12:56,481 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_51262. 2024-04-14T11:12:56,481 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_53461. 2024-04-14T11:12:56,482 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_55633. 2024-04-14T11:12:56,482 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_57727. 2024-04-14T11:12:56,482 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_59893. 2024-04-14T11:12:56,482 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_61998. 2024-04-14T11:12:56,483 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_64123. 2024-04-14T11:12:56,483 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_66293. 2024-04-14T11:12:56,483 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_68461. 2024-04-14T11:12:56,483 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_70603. 2024-04-14T11:12:56,484 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_72696. 2024-04-14T11:12:56,484 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_74923. 2024-04-14T11:12:56,484 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_77049. 2024-04-14T11:12:56,484 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_79166. 2024-04-14T11:12:56,484 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_81328. 2024-04-14T11:12:56,485 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_83471. 2024-04-14T11:12:56,485 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_85568. 2024-04-14T11:12:56,485 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_87731. 2024-04-14T11:12:56,486 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_89820. 2024-04-14T11:12:56,486 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_91916. 2024-04-14T11:12:56,486 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_94008. 2024-04-14T11:12:56,486 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_96118. 2024-04-14T11:12:56,487 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_98238. 2024-04-14T11:12:56,487 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_100429. 2024-04-14T11:12:56,488 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_102617. 2024-04-14T11:12:56,488 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_104913. 2024-04-14T11:12:56,488 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_107048. 2024-04-14T11:12:56,489 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_109218. 2024-04-14T11:12:56,489 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_111318. 2024-04-14T11:12:56,489 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_113405. 2024-04-14T11:12:56,489 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_115496. 2024-04-14T11:12:56,489 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_117596. 2024-04-14T11:12:56,490 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 0968972745424fb6b3a440a03184abc3_118657. 2024-04-14T11:12:56,490 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2064): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc221547-0085-fa7f-3168-33a7e0b8cd32/TestPrefetch2/0968972745424fb6b3a440a03184abc3, with total blocks 57, and data blocks 56 2024-04-14T11:12:56,490 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2068): iterating over 16 entries in the backing map 2024-04-14T11:12:56,490 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_14969 in the backing map. Acquiring read lock for offset 506880 2024-04-14T11:12:56,491 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_19170 in the backing map. Acquiring read lock for offset 491520 2024-04-14T11:12:56,491 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_55633 in the backing map. Acquiring read lock for offset 463872 2024-04-14T11:12:56,491 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_6425 in the backing map. Acquiring read lock for offset 516096 2024-04-14T11:12:56,491 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_10648 in the backing map. Acquiring read lock for offset 482304 2024-04-14T11:12:56,491 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_2129 in the backing map. Acquiring read lock for offset 522240 2024-04-14T11:12:56,491 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_51262 in the backing map. Acquiring read lock for offset 470016 2024-04-14T11:12:56,491 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_34147 in the backing map. Acquiring read lock for offset 476160 2024-04-14T11:12:56,491 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_29768 in the backing map. Acquiring read lock for offset 497664 2024-04-14T11:12:56,491 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_21263 in the backing map. Acquiring read lock for offset 503808 2024-04-14T11:12:56,492 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_25509 in the backing map. Acquiring read lock for offset 485376 2024-04-14T11:12:56,492 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_17059 in the backing map. Acquiring read lock for offset 479232 2024-04-14T11:12:56,492 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_8515 in the backing map. Acquiring read lock for offset 519168 2024-04-14T11:12:56,492 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_12797 in the backing map. Acquiring read lock for offset 509952 2024-04-14T11:12:56,492 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_53461 in the backing map. Acquiring read lock for offset 466944 2024-04-14T11:12:56,492 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_4302 in the backing map. Acquiring read lock for offset 513024 2024-04-14T11:12:56,492 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_0 in the backing map. Acquiring read lock for offset 525312 2024-04-14T11:12:56,492 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_49114 in the backing map. Acquiring read lock for offset 473088 2024-04-14T11:12:56,492 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_31907 in the backing map. Acquiring read lock for offset 494592 2024-04-14T11:12:56,492 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_23349 in the backing map. Acquiring read lock for offset 488448 2024-04-14T11:12:56,493 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_27614 in the backing map. Acquiring read lock for offset 500736 2024-04-14T11:12:56,493 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2096): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc221547-0085-fa7f-3168-33a7e0b8cd32/TestPrefetch2/0968972745424fb6b3a440a03184abc3, but only 21 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue. 2024-04-14T11:12:56,493 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2105): There are still blocks pending caching for file 0968972745424fb6b3a440a03184abc3. Will sleep 100ms and try the verification again. 2024-04-14T11:12:56,593 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2064): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc221547-0085-fa7f-3168-33a7e0b8cd32/TestPrefetch2/0968972745424fb6b3a440a03184abc3, with total blocks 57, and data blocks 56 2024-04-14T11:12:56,593 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2068): iterating over 57 entries in the backing map 2024-04-14T11:12:56,593 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_14969 in the backing map. Acquiring read lock for offset 506880 2024-04-14T11:12:56,593 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_64123 in the backing map. Acquiring read lock for offset 399360 2024-04-14T11:12:56,593 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_113405 in the backing map. Acquiring read lock for offset 368640 2024-04-14T11:12:56,594 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_19170 in the backing map. Acquiring read lock for offset 491520 2024-04-14T11:12:56,594 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_109218 in the backing map. Acquiring read lock for offset 356352 2024-04-14T11:12:56,594 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_55633 in the backing map. Acquiring read lock for offset 463872 2024-04-14T11:12:56,594 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_6425 in the backing map. Acquiring read lock for offset 516096 2024-04-14T11:12:56,594 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_59893 in the backing map. Acquiring read lock for offset 402432 2024-04-14T11:12:56,594 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_104913 in the backing map. Acquiring read lock for offset 359424 2024-04-14T11:12:56,594 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_10648 in the backing map. Acquiring read lock for offset 482304 2024-04-14T11:12:56,594 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_2129 in the backing map. Acquiring read lock for offset 522240 2024-04-14T11:12:56,594 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_100429 in the backing map. Acquiring read lock for offset 380928 2024-04-14T11:12:56,594 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_51262 in the backing map. Acquiring read lock for offset 470016 2024-04-14T11:12:56,595 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_96118 in the backing map. Acquiring read lock for offset 384000 2024-04-14T11:12:56,595 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_46949 in the backing map. Acquiring read lock for offset 454656 2024-04-14T11:12:56,595 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_83471 in the backing map. Acquiring read lock for offset 436224 2024-04-14T11:12:56,595 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_91916 in the backing map. Acquiring read lock for offset 423936 2024-04-14T11:12:56,595 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_87731 in the backing map. Acquiring read lock for offset 427008 2024-04-14T11:12:56,595 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_42653 in the backing map. Acquiring read lock for offset 457728 2024-04-14T11:12:56,595 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_34147 in the backing map. Acquiring read lock for offset 476160 2024-04-14T11:12:56,595 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_79166 in the backing map. Acquiring read lock for offset 442368 2024-04-14T11:12:56,595 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_38334 in the backing map. Acquiring read lock for offset 451584 2024-04-14T11:12:56,596 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_29768 in the backing map. Acquiring read lock for offset 497664 2024-04-14T11:12:56,596 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_74923 in the backing map. Acquiring read lock for offset 390144 2024-04-14T11:12:56,596 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_115496 in the backing map. Acquiring read lock for offset 365568 2024-04-14T11:12:56,596 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_21263 in the backing map. Acquiring read lock for offset 503808 2024-04-14T11:12:56,596 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_70603 in the backing map. Acquiring read lock for offset 393216 2024-04-14T11:12:56,596 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_25509 in the backing map. Acquiring read lock for offset 485376 2024-04-14T11:12:56,596 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_61998 in the backing map. Acquiring read lock for offset 430080 2024-04-14T11:12:56,596 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_107048 in the backing map. Acquiring read lock for offset 371712 2024-04-14T11:12:56,597 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_66293 in the backing map. Acquiring read lock for offset 445440 2024-04-14T11:12:56,597 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_111318 in the backing map. Acquiring read lock for offset 353280 2024-04-14T11:12:56,597 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_17059 in the backing map. Acquiring read lock for offset 479232 2024-04-14T11:12:56,597 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_57727 in the backing map. Acquiring read lock for offset 405504 2024-04-14T11:12:56,597 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_8515 in the backing map. Acquiring read lock for offset 519168 2024-04-14T11:12:56,597 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_12797 in the backing map. Acquiring read lock for offset 509952 2024-04-14T11:12:56,597 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_53461 in the backing map. Acquiring read lock for offset 466944 2024-04-14T11:12:56,597 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_102617 in the backing map. Acquiring read lock for offset 374784 2024-04-14T11:12:56,598 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_4302 in the backing map. Acquiring read lock for offset 513024 2024-04-14T11:12:56,598 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_94008 in the backing map. Acquiring read lock for offset 411648 2024-04-14T11:12:56,598 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_0 in the backing map. Acquiring read lock for offset 525312 2024-04-14T11:12:56,598 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_49114 in the backing map. Acquiring read lock for offset 473088 2024-04-14T11:12:56,598 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_98238 in the backing map. Acquiring read lock for offset 408576 2024-04-14T11:12:56,598 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_85568 in the backing map. Acquiring read lock for offset 433152 2024-04-14T11:12:56,598 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_44790 in the backing map. Acquiring read lock for offset 439296 2024-04-14T11:12:56,598 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_89820 in the backing map. Acquiring read lock for offset 414720 2024-04-14T11:12:56,598 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_40437 in the backing map. Acquiring read lock for offset 448512 2024-04-14T11:12:56,599 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_81328 in the backing map. Acquiring read lock for offset 387072 2024-04-14T11:12:56,599 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_36243 in the backing map. Acquiring read lock for offset 460800 2024-04-14T11:12:56,599 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_77049 in the backing map. Acquiring read lock for offset 417792 2024-04-14T11:12:56,599 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_31907 in the backing map. Acquiring read lock for offset 494592 2024-04-14T11:12:56,599 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_68461 in the backing map. Acquiring read lock for offset 396288 2024-04-14T11:12:56,599 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_117596 in the backing map. Acquiring read lock for offset 362496 2024-04-14T11:12:56,599 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_23349 in the backing map. Acquiring read lock for offset 488448 2024-04-14T11:12:56,599 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_72696 in the backing map. Acquiring read lock for offset 420864 2024-04-14T11:12:56,599 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block 0968972745424fb6b3a440a03184abc3_27614 in the backing map. Acquiring read lock for offset 500736 2024-04-14T11:12:56,600 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2093): File /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc221547-0085-fa7f-3168-33a7e0b8cd32/TestPrefetch2/0968972745424fb6b3a440a03184abc3 has now been fully cached. 2024-04-14T11:12:56,600 WARN [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(151): Prefetch path=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc221547-0085-fa7f-3168-33a7e0b8cd32/TestPrefetch2/0968972745424fb6b3a440a03184abc3, offset=118950, end=118950 java.lang.IllegalMonitorStateException: attempt to unlock read lock, not locked by current thread at java.util.concurrent.locks.ReentrantReadWriteLock$Sync.unmatchedUnlockException(ReentrantReadWriteLock.java:444) ~[?:1.8.0_362] at java.util.concurrent.locks.ReentrantReadWriteLock$Sync.tryReleaseShared(ReentrantReadWriteLock.java:428) ~[?:1.8.0_362] at java.util.concurrent.locks.AbstractQueuedSynchronizer.releaseShared(AbstractQueuedSynchronizer.java:1341) ~[?:1.8.0_362] at java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.unlock(ReentrantReadWriteLock.java:881) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2125) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:141) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_362] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) ~[?:1.8.0_362] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:12:56,600 DEBUG [hfile-prefetch-1713093174282 {}] hfile.PrefetchExecutor(108): Prefetch completed for 0968972745424fb6b3a440a03184abc3 2024-04-14T11:12:57,577 INFO [Time-limited test {}] bucket.BucketCache(1534): Disabling cache 2024-04-14T11:12:57,577 INFO [Time-limited test {}] bucket.BucketCache(1558): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc221547-0085-fa7f-3168-33a7e0b8cd32/bucket.persistence 2024-04-14T11:12:57,577 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1094): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-04-14T11:12:57,577 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1094): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-04-14T11:12:57,577 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1094): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-04-14T11:12:57,577 INFO [Time-limited test {}] bucket.BucketCache(1564): Shutting down cache persister thread. 2024-04-14T11:12:57,578 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-04-14T11:12:57,588 DEBUG [Time-limited test {}] bucket.BucketCache(1310): Thread Time-limited test started persisting bucket cache to file 2024-04-14T11:12:57,594 DEBUG [Time-limited test {}] bucket.BucketCache(1323): Thread Time-limited test finished persisting bucket cache to file, renaming 2024-04-14T11:12:57,595 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-04-14T11:12:57,596 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-04-14T11:12:57,607 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestBucketCachePersister#testPrefetchPersistenceCrashNegative Thread=15 (was 14) Potentially hanging thread: hfile-prefetch-1713093175573 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=309 (was 308) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=229 (was 229), ProcessCount=170 (was 170), AvailableMemoryMB=3190 (was 3195) 2024-04-14T11:12:57,621 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestBucketCachePersister#testPrefetchListUponBlockEviction Thread=15, OpenFileDescriptor=309, MaxFileDescriptor=60000, SystemLoadAverage=229, ProcessCount=170, AvailableMemoryMB=3190 2024-04-14T11:12:57,621 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-04-14T11:12:57,622 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc221547-0085-fa7f-3168-33a7e0b8cd32/testPrefetchListUponBlockEviction 2024-04-14T11:12:57,622 INFO [Time-limited test {}] bucket.BucketCache(305): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2024-04-14T11:12:57,622 INFO [Time-limited test {}] bucket.BucketCache(393): Starting BucketCachePersisterThread 2024-04-14T11:12:57,623 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 200 2024-04-14T11:12:57,623 INFO [Time-limited test {}] bucket.BucketCache(1344): Started retrieving bucket cache from file 2024-04-14T11:12:57,623 WARN [Time-limited test {}] bucket.BucketCache(1347): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-04-14T11:12:57,623 INFO [Time-limited test {}] bucket.BucketAllocator(339): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-04-14T11:12:57,628 INFO [Time-limited test {}] bucket.BucketCache(356): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc221547-0085-fa7f-3168-33a7e0b8cd32/testPrefetchListUponBlockEviction, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc221547-0085-fa7f-3168-33a7e0b8cd32/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-04-14T11:12:57,628 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-04-14T11:12:57,632 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b48e848ae74249c385cf57dce4c5a760 2024-04-14T11:12:57,642 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(813): Len of the biggest cell in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc221547-0085-fa7f-3168-33a7e0b8cd32/TestPrefetch3/b48e848ae74249c385cf57dce4c5a760 is 252, key is aaaaaaaaaaaaaaaaaaaaaaaaaababaaa/Bd2MAQEu/-3434109585807246770/Put/seqid=0 2024-04-14T11:12:57,644 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(92): Prefetch requested for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc221547-0085-fa7f-3168-33a7e0b8cd32/TestPrefetch3/b48e848ae74249c385cf57dce4c5a760, delay=980 ms 2024-04-14T11:12:57,823 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-04-14T11:12:57,823 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1310): Thread bucket-cache-persister started persisting bucket cache to file 2024-04-14T11:12:57,830 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1323): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-04-14T11:12:58,625 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_0. 2024-04-14T11:12:58,626 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_2173. 2024-04-14T11:12:58,627 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_4272. 2024-04-14T11:12:58,627 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_6457. 2024-04-14T11:12:58,627 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_8651. 2024-04-14T11:12:58,628 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_10788. 2024-04-14T11:12:58,628 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_12944. 2024-04-14T11:12:58,628 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_15128. 2024-04-14T11:12:58,629 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_17321. 2024-04-14T11:12:58,629 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_19458. 2024-04-14T11:12:58,629 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_21624. 2024-04-14T11:12:58,629 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_23714. 2024-04-14T11:12:58,630 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_25834. 2024-04-14T11:12:58,630 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_27989. 2024-04-14T11:12:58,630 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_30152. 2024-04-14T11:12:58,631 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_32273. 2024-04-14T11:12:58,631 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_34456. 2024-04-14T11:12:58,631 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_36584. 2024-04-14T11:12:58,631 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_38755. 2024-04-14T11:12:58,632 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_40916. 2024-04-14T11:12:58,632 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_43062. 2024-04-14T11:12:58,632 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_45153. 2024-04-14T11:12:58,632 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_47305. 2024-04-14T11:12:58,632 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_49393. 2024-04-14T11:12:58,632 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_51605. 2024-04-14T11:12:58,633 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_53721. 2024-04-14T11:12:58,633 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_55807. 2024-04-14T11:12:58,633 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_57931. 2024-04-14T11:12:58,633 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_60057. 2024-04-14T11:12:58,634 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_62161. 2024-04-14T11:12:58,634 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_64326. 2024-04-14T11:12:58,634 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_66449. 2024-04-14T11:12:58,634 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_68615. 2024-04-14T11:12:58,635 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_70772. 2024-04-14T11:12:58,635 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_72945. 2024-04-14T11:12:58,635 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_75106. 2024-04-14T11:12:58,635 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_77209. 2024-04-14T11:12:58,635 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_79316. 2024-04-14T11:12:58,635 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_81446. 2024-04-14T11:12:58,636 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_83567. 2024-04-14T11:12:58,636 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_85762. 2024-04-14T11:12:58,636 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_87979. 2024-04-14T11:12:58,636 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_90098. 2024-04-14T11:12:58,636 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_92295. 2024-04-14T11:12:58,637 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_94454. 2024-04-14T11:12:58,637 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_96663. 2024-04-14T11:12:58,638 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_98832. 2024-04-14T11:12:58,641 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_100960. 2024-04-14T11:12:58,641 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_103052. 2024-04-14T11:12:58,642 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_105169. 2024-04-14T11:12:58,642 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_107307. 2024-04-14T11:12:58,642 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_109421. 2024-04-14T11:12:58,642 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_111547. 2024-04-14T11:12:58,643 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_113712. 2024-04-14T11:12:58,643 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_115892. 2024-04-14T11:12:58,643 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_118073. 2024-04-14T11:12:58,643 DEBUG [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b48e848ae74249c385cf57dce4c5a760_118864. 2024-04-14T11:12:58,644 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2064): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc221547-0085-fa7f-3168-33a7e0b8cd32/TestPrefetch3/b48e848ae74249c385cf57dce4c5a760, with total blocks 57, and data blocks 56 2024-04-14T11:12:58,644 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2068): iterating over 13 entries in the backing map 2024-04-14T11:12:58,644 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_0 in the backing map. Acquiring read lock for offset 525312 2024-04-14T11:12:58,644 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_4272 in the backing map. Acquiring read lock for offset 513024 2024-04-14T11:12:58,644 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_8651 in the backing map. Acquiring read lock for offset 519168 2024-04-14T11:12:58,644 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_12944 in the backing map. Acquiring read lock for offset 494592 2024-04-14T11:12:58,644 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_17321 in the backing map. Acquiring read lock for offset 488448 2024-04-14T11:12:58,644 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_25834 in the backing map. Acquiring read lock for offset 509952 2024-04-14T11:12:58,644 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_21624 in the backing map. Acquiring read lock for offset 497664 2024-04-14T11:12:58,644 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_34456 in the backing map. Acquiring read lock for offset 503808 2024-04-14T11:12:58,644 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_2173 in the backing map. Acquiring read lock for offset 522240 2024-04-14T11:12:58,644 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_6457 in the backing map. Acquiring read lock for offset 516096 2024-04-14T11:12:58,645 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_15128 in the backing map. Acquiring read lock for offset 491520 2024-04-14T11:12:58,645 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_10788 in the backing map. Acquiring read lock for offset 506880 2024-04-14T11:12:58,645 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_19458 in the backing map. Acquiring read lock for offset 500736 2024-04-14T11:12:58,645 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2096): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc221547-0085-fa7f-3168-33a7e0b8cd32/TestPrefetch3/b48e848ae74249c385cf57dce4c5a760, but only 13 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue. 2024-04-14T11:12:58,645 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2105): There are still blocks pending caching for file b48e848ae74249c385cf57dce4c5a760. Will sleep 100ms and try the verification again. 2024-04-14T11:12:58,745 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2064): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc221547-0085-fa7f-3168-33a7e0b8cd32/TestPrefetch3/b48e848ae74249c385cf57dce4c5a760, with total blocks 57, and data blocks 56 2024-04-14T11:12:58,745 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2068): iterating over 57 entries in the backing map 2024-04-14T11:12:58,746 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_36584 in the backing map. Acquiring read lock for offset 473088 2024-04-14T11:12:58,746 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_85762 in the backing map. Acquiring read lock for offset 427008 2024-04-14T11:12:58,746 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_81446 in the backing map. Acquiring read lock for offset 399360 2024-04-14T11:12:58,746 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_90098 in the backing map. Acquiring read lock for offset 393216 2024-04-14T11:12:58,746 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_0 in the backing map. Acquiring read lock for offset 525312 2024-04-14T11:12:58,746 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_40916 in the backing map. Acquiring read lock for offset 470016 2024-04-14T11:12:58,746 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_94454 in the backing map. Acquiring read lock for offset 390144 2024-04-14T11:12:58,746 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_49393 in the backing map. Acquiring read lock for offset 448512 2024-04-14T11:12:58,746 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_4272 in the backing map. Acquiring read lock for offset 513024 2024-04-14T11:12:58,746 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_45153 in the backing map. Acquiring read lock for offset 454656 2024-04-14T11:12:58,746 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_98832 in the backing map. Acquiring read lock for offset 368640 2024-04-14T11:12:58,746 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_53721 in the backing map. Acquiring read lock for offset 451584 2024-04-14T11:12:58,747 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_8651 in the backing map. Acquiring read lock for offset 519168 2024-04-14T11:12:58,747 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_62161 in the backing map. Acquiring read lock for offset 436224 2024-04-14T11:12:58,747 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_12944 in the backing map. Acquiring read lock for offset 494592 2024-04-14T11:12:58,747 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_103052 in the backing map. Acquiring read lock for offset 387072 2024-04-14T11:12:58,747 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_57931 in the backing map. Acquiring read lock for offset 439296 2024-04-14T11:12:58,747 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_17321 in the backing map. Acquiring read lock for offset 488448 2024-04-14T11:12:58,747 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_111547 in the backing map. Acquiring read lock for offset 359424 2024-04-14T11:12:58,747 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_66449 in the backing map. Acquiring read lock for offset 411648 2024-04-14T11:12:58,747 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_107307 in the backing map. Acquiring read lock for offset 362496 2024-04-14T11:12:58,747 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_25834 in the backing map. Acquiring read lock for offset 509952 2024-04-14T11:12:58,747 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_115892 in the backing map. Acquiring read lock for offset 380928 2024-04-14T11:12:58,748 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_70772 in the backing map. Acquiring read lock for offset 405504 2024-04-14T11:12:58,748 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_21624 in the backing map. Acquiring read lock for offset 497664 2024-04-14T11:12:58,748 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_30152 in the backing map. Acquiring read lock for offset 479232 2024-04-14T11:12:58,748 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_79316 in the backing map. Acquiring read lock for offset 402432 2024-04-14T11:12:58,748 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_75106 in the backing map. Acquiring read lock for offset 433152 2024-04-14T11:12:58,748 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_83567 in the backing map. Acquiring read lock for offset 396288 2024-04-14T11:12:58,748 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_34456 in the backing map. Acquiring read lock for offset 503808 2024-04-14T11:12:58,748 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_87979 in the backing map. Acquiring read lock for offset 423936 2024-04-14T11:12:58,748 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_38755 in the backing map. Acquiring read lock for offset 463872 2024-04-14T11:12:58,748 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_47305 in the backing map. Acquiring read lock for offset 457728 2024-04-14T11:12:58,748 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_92295 in the backing map. Acquiring read lock for offset 414720 2024-04-14T11:12:58,749 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_2173 in the backing map. Acquiring read lock for offset 522240 2024-04-14T11:12:58,749 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_43062 in the backing map. Acquiring read lock for offset 460800 2024-04-14T11:12:58,749 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_55807 in the backing map. Acquiring read lock for offset 442368 2024-04-14T11:12:58,749 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_51605 in the backing map. Acquiring read lock for offset 466944 2024-04-14T11:12:58,749 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_96663 in the backing map. Acquiring read lock for offset 374784 2024-04-14T11:12:58,749 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_6457 in the backing map. Acquiring read lock for offset 516096 2024-04-14T11:12:58,749 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_15128 in the backing map. Acquiring read lock for offset 491520 2024-04-14T11:12:58,749 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_105169 in the backing map. Acquiring read lock for offset 365568 2024-04-14T11:12:58,749 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_100960 in the backing map. Acquiring read lock for offset 371712 2024-04-14T11:12:58,749 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_60057 in the backing map. Acquiring read lock for offset 445440 2024-04-14T11:12:58,749 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_10788 in the backing map. Acquiring read lock for offset 506880 2024-04-14T11:12:58,749 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_19458 in the backing map. Acquiring read lock for offset 500736 2024-04-14T11:12:58,749 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_68615 in the backing map. Acquiring read lock for offset 408576 2024-04-14T11:12:58,749 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_109421 in the backing map. Acquiring read lock for offset 384000 2024-04-14T11:12:58,750 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_64326 in the backing map. Acquiring read lock for offset 420864 2024-04-14T11:12:58,750 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_72945 in the backing map. Acquiring read lock for offset 417792 2024-04-14T11:12:58,750 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_23714 in the backing map. Acquiring read lock for offset 485376 2024-04-14T11:12:58,750 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_113712 in the backing map. Acquiring read lock for offset 356352 2024-04-14T11:12:58,750 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_32273 in the backing map. Acquiring read lock for offset 476160 2024-04-14T11:12:58,750 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_77209 in the backing map. Acquiring read lock for offset 430080 2024-04-14T11:12:58,750 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_118073 in the backing map. Acquiring read lock for offset 377856 2024-04-14T11:12:58,751 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2075): found block b48e848ae74249c385cf57dce4c5a760_27989 in the backing map. Acquiring read lock for offset 482304 2024-04-14T11:12:58,751 DEBUG [hfile-prefetch-1713093175573 {}] bucket.BucketCache(2093): File /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc221547-0085-fa7f-3168-33a7e0b8cd32/TestPrefetch3/b48e848ae74249c385cf57dce4c5a760 has now been fully cached. 2024-04-14T11:12:58,751 WARN [hfile-prefetch-1713093175573 {}] hfile.HFilePreadReader$1(151): Prefetch path=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc221547-0085-fa7f-3168-33a7e0b8cd32/TestPrefetch3/b48e848ae74249c385cf57dce4c5a760, offset=119157, end=119157 java.lang.IllegalMonitorStateException: attempt to unlock read lock, not locked by current thread at java.util.concurrent.locks.ReentrantReadWriteLock$Sync.unmatchedUnlockException(ReentrantReadWriteLock.java:444) ~[?:1.8.0_362] at java.util.concurrent.locks.ReentrantReadWriteLock$Sync.tryReleaseShared(ReentrantReadWriteLock.java:428) ~[?:1.8.0_362] at java.util.concurrent.locks.AbstractQueuedSynchronizer.releaseShared(AbstractQueuedSynchronizer.java:1341) ~[?:1.8.0_362] at java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.unlock(ReentrantReadWriteLock.java:881) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2125) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:141) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_362] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) ~[?:1.8.0_362] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:12:58,752 DEBUG [hfile-prefetch-1713093175573 {}] hfile.PrefetchExecutor(108): Prefetch completed for b48e848ae74249c385cf57dce4c5a760 2024-04-14T11:12:58,831 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-04-14T11:12:58,831 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1310): Thread bucket-cache-persister started persisting bucket cache to file 2024-04-14T11:12:58,836 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1323): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-04-14T11:12:59,646 DEBUG [Time-limited test {}] bucket.BucketCache(739): removed key b48e848ae74249c385cf57dce4c5a760_36584 from back map with offset lock 473088 in the evict process 2024-04-14T11:12:59,647 DEBUG [Time-limited test {}] bucket.BucketCache(651): Removing file b48e848ae74249c385cf57dce4c5a760 for region cc221547-0085-fa7f-3168-33a7e0b8cd32 2024-04-14T11:12:59,647 INFO [Time-limited test {}] bucket.BucketCache(1534): Disabling cache 2024-04-14T11:12:59,648 INFO [Time-limited test {}] bucket.BucketCache(1558): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc221547-0085-fa7f-3168-33a7e0b8cd32/bucket.persistence 2024-04-14T11:12:59,648 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1094): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-04-14T11:12:59,648 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1094): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-04-14T11:12:59,648 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1094): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-04-14T11:12:59,648 INFO [Time-limited test {}] bucket.BucketCache(1564): Shutting down cache persister thread. 2024-04-14T11:12:59,649 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-04-14T11:12:59,659 DEBUG [Time-limited test {}] bucket.BucketCache(1310): Thread Time-limited test started persisting bucket cache to file 2024-04-14T11:12:59,664 DEBUG [Time-limited test {}] bucket.BucketCache(1323): Thread Time-limited test finished persisting bucket cache to file, renaming 2024-04-14T11:12:59,665 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-04-14T11:12:59,666 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-04-14T11:12:59,680 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestBucketCachePersister#testPrefetchListUponBlockEviction Thread=16 (was 15) Potentially hanging thread: hfile-prefetch-1713093177644 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=310 (was 309) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=236 (was 229) - SystemLoadAverage LEAK? -, ProcessCount=170 (was 170), AvailableMemoryMB=3129 (was 3190) 2024-04-14T11:12:59,691 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestBucketCachePersister#testPrefetchPersistenceCrash Thread=16, OpenFileDescriptor=310, MaxFileDescriptor=60000, SystemLoadAverage=236, ProcessCount=170, AvailableMemoryMB=3129 2024-04-14T11:12:59,691 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-04-14T11:12:59,692 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc221547-0085-fa7f-3168-33a7e0b8cd32/testPrefetchPersistenceCrash 2024-04-14T11:12:59,692 INFO [Time-limited test {}] bucket.BucketCache(305): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2024-04-14T11:12:59,692 INFO [Time-limited test {}] bucket.BucketCache(393): Starting BucketCachePersisterThread 2024-04-14T11:12:59,692 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 3000 2024-04-14T11:12:59,693 INFO [Time-limited test {}] bucket.BucketCache(1344): Started retrieving bucket cache from file 2024-04-14T11:12:59,693 WARN [Time-limited test {}] bucket.BucketCache(1347): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-04-14T11:12:59,693 INFO [Time-limited test {}] bucket.BucketAllocator(339): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-04-14T11:12:59,694 INFO [Time-limited test {}] bucket.BucketCache(356): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc221547-0085-fa7f-3168-33a7e0b8cd32/testPrefetchPersistenceCrash, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc221547-0085-fa7f-3168-33a7e0b8cd32/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-04-14T11:12:59,694 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-04-14T11:12:59,698 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d2d08d6c8bbf47abb7e98f55d1f331de 2024-04-14T11:12:59,706 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(813): Len of the biggest cell in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc221547-0085-fa7f-3168-33a7e0b8cd32/TestPrefetch0/d2d08d6c8bbf47abb7e98f55d1f331de is 225, key is aaaaaaaaaaaaaaaaaaaaaabbababbbaa/lEPQ0:v/-6458371555009016951/Put/seqid=0 2024-04-14T11:12:59,711 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on cf50821fedfd4f098e31b9522dcbb320 2024-04-14T11:12:59,719 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(813): Len of the biggest cell in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc221547-0085-fa7f-3168-33a7e0b8cd32/TestPrefetch1/cf50821fedfd4f098e31b9522dcbb320 is 243, key is aaaaaaaaaaaaaaaaaaaaaabbabbabbba/5R__l:7/-1709550343867591743/Put/seqid=0 2024-04-14T11:12:59,721 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(92): Prefetch requested for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc221547-0085-fa7f-3168-33a7e0b8cd32/TestPrefetch0/d2d08d6c8bbf47abb7e98f55d1f331de, delay=991 ms 2024-04-14T11:13:00,713 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_0. 2024-04-14T11:13:00,714 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_2093. 2024-04-14T11:13:00,714 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_4220. 2024-04-14T11:13:00,715 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_6339. 2024-04-14T11:13:00,715 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_8499. 2024-04-14T11:13:00,715 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_10689. 2024-04-14T11:13:00,715 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_12781. 2024-04-14T11:13:00,716 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_14890. 2024-04-14T11:13:00,716 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_17089. 2024-04-14T11:13:00,716 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_19241. 2024-04-14T11:13:00,716 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_21358. 2024-04-14T11:13:00,716 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_23518. 2024-04-14T11:13:00,717 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_25642. 2024-04-14T11:13:00,717 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_27757. 2024-04-14T11:13:00,717 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_29975. 2024-04-14T11:13:00,718 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_32131. 2024-04-14T11:13:00,718 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_34261. 2024-04-14T11:13:00,718 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_36397. 2024-04-14T11:13:00,718 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_38543. 2024-04-14T11:13:00,719 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_40712. 2024-04-14T11:13:00,719 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_42906. 2024-04-14T11:13:00,719 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_44994. 2024-04-14T11:13:00,719 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_47088. 2024-04-14T11:13:00,719 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_49306. 2024-04-14T11:13:00,720 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_51430. 2024-04-14T11:13:00,720 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_53634. 2024-04-14T11:13:00,720 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_55755. 2024-04-14T11:13:00,720 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_57937. 2024-04-14T11:13:00,720 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_60119. 2024-04-14T11:13:00,720 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_62303. 2024-04-14T11:13:00,721 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_64442. 2024-04-14T11:13:00,721 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_66554. 2024-04-14T11:13:00,721 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_68643. 2024-04-14T11:13:00,721 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_70775. 2024-04-14T11:13:00,721 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_73008. 2024-04-14T11:13:00,721 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_75171. 2024-04-14T11:13:00,722 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_77403. 2024-04-14T11:13:00,722 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_79578. 2024-04-14T11:13:00,722 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_81756. 2024-04-14T11:13:00,722 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_83904. 2024-04-14T11:13:00,722 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_86042. 2024-04-14T11:13:00,722 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_88132. 2024-04-14T11:13:00,723 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_90260. 2024-04-14T11:13:00,723 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_92380. 2024-04-14T11:13:00,723 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_94522. 2024-04-14T11:13:00,723 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_96612. 2024-04-14T11:13:00,723 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_98738. 2024-04-14T11:13:00,723 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_100875. 2024-04-14T11:13:00,724 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_103103. 2024-04-14T11:13:00,724 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_105229. 2024-04-14T11:13:00,724 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_107426. 2024-04-14T11:13:00,724 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_109641. 2024-04-14T11:13:00,724 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_111829. 2024-04-14T11:13:00,725 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_113969. 2024-04-14T11:13:00,725 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_116054. 2024-04-14T11:13:00,725 DEBUG [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key d2d08d6c8bbf47abb7e98f55d1f331de_118220. 2024-04-14T11:13:00,725 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2064): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc221547-0085-fa7f-3168-33a7e0b8cd32/TestPrefetch0/d2d08d6c8bbf47abb7e98f55d1f331de, with total blocks 56, and data blocks 55 2024-04-14T11:13:00,725 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2068): iterating over 3 entries in the backing map 2024-04-14T11:13:00,725 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_12781 in the backing map. Acquiring read lock for offset 519168 2024-04-14T11:13:00,725 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_2093 in the backing map. Acquiring read lock for offset 522240 2024-04-14T11:13:00,725 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_0 in the backing map. Acquiring read lock for offset 525312 2024-04-14T11:13:00,725 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2096): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc221547-0085-fa7f-3168-33a7e0b8cd32/TestPrefetch0/d2d08d6c8bbf47abb7e98f55d1f331de, but only 3 data blocks were cached. Total data blocks for file: 55. Checking for blocks pending cache in cache writer queue. 2024-04-14T11:13:00,725 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2105): There are still blocks pending caching for file d2d08d6c8bbf47abb7e98f55d1f331de. Will sleep 100ms and try the verification again. 2024-04-14T11:13:00,826 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2064): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc221547-0085-fa7f-3168-33a7e0b8cd32/TestPrefetch0/d2d08d6c8bbf47abb7e98f55d1f331de, with total blocks 56, and data blocks 55 2024-04-14T11:13:00,826 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2068): iterating over 56 entries in the backing map 2024-04-14T11:13:00,826 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_29975 in the backing map. Acquiring read lock for offset 503808 2024-04-14T11:13:00,826 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_116054 in the backing map. Acquiring read lock for offset 368640 2024-04-14T11:13:00,826 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_111829 in the backing map. Acquiring read lock for offset 423936 2024-04-14T11:13:00,826 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_25642 in the backing map. Acquiring read lock for offset 485376 2024-04-14T11:13:00,826 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_70775 in the backing map. Acquiring read lock for offset 405504 2024-04-14T11:13:00,826 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_107426 in the backing map. Acquiring read lock for offset 380928 2024-04-14T11:13:00,826 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_66554 in the backing map. Acquiring read lock for offset 411648 2024-04-14T11:13:00,827 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_21358 in the backing map. Acquiring read lock for offset 491520 2024-04-14T11:13:00,827 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_62303 in the backing map. Acquiring read lock for offset 427008 2024-04-14T11:13:00,827 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_103103 in the backing map. Acquiring read lock for offset 384000 2024-04-14T11:13:00,827 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_17089 in the backing map. Acquiring read lock for offset 494592 2024-04-14T11:13:00,827 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_57937 in the backing map. Acquiring read lock for offset 463872 2024-04-14T11:13:00,827 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_98738 in the backing map. Acquiring read lock for offset 390144 2024-04-14T11:13:00,827 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_12781 in the backing map. Acquiring read lock for offset 519168 2024-04-14T11:13:00,827 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_96612 in the backing map. Acquiring read lock for offset 362496 2024-04-14T11:13:00,827 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_6339 in the backing map. Acquiring read lock for offset 513024 2024-04-14T11:13:00,827 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_51430 in the backing map. Acquiring read lock for offset 466944 2024-04-14T11:13:00,827 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_92380 in the backing map. Acquiring read lock for offset 457728 2024-04-14T11:13:00,827 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_88132 in the backing map. Acquiring read lock for offset 460800 2024-04-14T11:13:00,828 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_2093 in the backing map. Acquiring read lock for offset 522240 2024-04-14T11:13:00,828 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_83904 in the backing map. Acquiring read lock for offset 371712 2024-04-14T11:13:00,828 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_42906 in the backing map. Acquiring read lock for offset 451584 2024-04-14T11:13:00,828 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_47088 in the backing map. Acquiring read lock for offset 439296 2024-04-14T11:13:00,828 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_79578 in the backing map. Acquiring read lock for offset 387072 2024-04-14T11:13:00,828 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_38543 in the backing map. Acquiring read lock for offset 448512 2024-04-14T11:13:00,828 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_75171 in the backing map. Acquiring read lock for offset 402432 2024-04-14T11:13:00,828 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_34261 in the backing map. Acquiring read lock for offset 476160 2024-04-14T11:13:00,828 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_73008 in the backing map. Acquiring read lock for offset 408576 2024-04-14T11:13:00,828 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_113969 in the backing map. Acquiring read lock for offset 374784 2024-04-14T11:13:00,828 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_32131 in the backing map. Acquiring read lock for offset 479232 2024-04-14T11:13:00,828 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_109641 in the backing map. Acquiring read lock for offset 356352 2024-04-14T11:13:00,829 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_68643 in the backing map. Acquiring read lock for offset 414720 2024-04-14T11:13:00,829 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_27757 in the backing map. Acquiring read lock for offset 482304 2024-04-14T11:13:00,829 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_64442 in the backing map. Acquiring read lock for offset 417792 2024-04-14T11:13:00,829 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_23518 in the backing map. Acquiring read lock for offset 488448 2024-04-14T11:13:00,829 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_19241 in the backing map. Acquiring read lock for offset 509952 2024-04-14T11:13:00,829 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_105229 in the backing map. Acquiring read lock for offset 433152 2024-04-14T11:13:00,829 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_60119 in the backing map. Acquiring read lock for offset 436224 2024-04-14T11:13:00,829 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_14890 in the backing map. Acquiring read lock for offset 497664 2024-04-14T11:13:00,829 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_55755 in the backing map. Acquiring read lock for offset 420864 2024-04-14T11:13:00,829 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_10689 in the backing map. Acquiring read lock for offset 500736 2024-04-14T11:13:00,829 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_100875 in the backing map. Acquiring read lock for offset 359424 2024-04-14T11:13:00,829 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_94522 in the backing map. Acquiring read lock for offset 454656 2024-04-14T11:13:00,830 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_8499 in the backing map. Acquiring read lock for offset 506880 2024-04-14T11:13:00,830 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_53634 in the backing map. Acquiring read lock for offset 430080 2024-04-14T11:13:00,830 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_49306 in the backing map. Acquiring read lock for offset 470016 2024-04-14T11:13:00,830 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_86042 in the backing map. Acquiring read lock for offset 399360 2024-04-14T11:13:00,830 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_90260 in the backing map. Acquiring read lock for offset 396288 2024-04-14T11:13:00,830 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_4220 in the backing map. Acquiring read lock for offset 516096 2024-04-14T11:13:00,830 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_44994 in the backing map. Acquiring read lock for offset 445440 2024-04-14T11:13:00,830 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_0 in the backing map. Acquiring read lock for offset 525312 2024-04-14T11:13:00,830 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_81756 in the backing map. Acquiring read lock for offset 377856 2024-04-14T11:13:00,830 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_40712 in the backing map. Acquiring read lock for offset 442368 2024-04-14T11:13:00,830 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_36397 in the backing map. Acquiring read lock for offset 473088 2024-04-14T11:13:00,830 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2075): found block d2d08d6c8bbf47abb7e98f55d1f331de_77403 in the backing map. Acquiring read lock for offset 393216 2024-04-14T11:13:00,830 DEBUG [hfile-prefetch-1713093174282 {}] bucket.BucketCache(2093): File /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc221547-0085-fa7f-3168-33a7e0b8cd32/TestPrefetch0/d2d08d6c8bbf47abb7e98f55d1f331de has now been fully cached. 2024-04-14T11:13:00,831 WARN [hfile-prefetch-1713093174282 {}] hfile.HFilePreadReader$1(151): Prefetch path=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc221547-0085-fa7f-3168-33a7e0b8cd32/TestPrefetch0/d2d08d6c8bbf47abb7e98f55d1f331de, offset=118513, end=118513 java.lang.IllegalMonitorStateException: attempt to unlock read lock, not locked by current thread at java.util.concurrent.locks.ReentrantReadWriteLock$Sync.unmatchedUnlockException(ReentrantReadWriteLock.java:444) ~[?:1.8.0_362] at java.util.concurrent.locks.ReentrantReadWriteLock$Sync.tryReleaseShared(ReentrantReadWriteLock.java:428) ~[?:1.8.0_362] at java.util.concurrent.locks.AbstractQueuedSynchronizer.releaseShared(AbstractQueuedSynchronizer.java:1341) ~[?:1.8.0_362] at java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.unlock(ReentrantReadWriteLock.java:881) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2125) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:141) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_362] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) ~[?:1.8.0_362] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:13:00,831 DEBUG [hfile-prefetch-1713093174282 {}] hfile.PrefetchExecutor(108): Prefetch completed for d2d08d6c8bbf47abb7e98f55d1f331de 2024-04-14T11:13:01,724 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(92): Prefetch requested for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc221547-0085-fa7f-3168-33a7e0b8cd32/TestPrefetch1/cf50821fedfd4f098e31b9522dcbb320, delay=950 ms 2024-04-14T11:13:02,675 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_0. 2024-04-14T11:13:02,676 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_2197. 2024-04-14T11:13:02,676 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_4395. 2024-04-14T11:13:02,676 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_6559. 2024-04-14T11:13:02,677 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_8674. 2024-04-14T11:13:02,677 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_10764. 2024-04-14T11:13:02,677 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_12856. 2024-04-14T11:13:02,677 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_15036. 2024-04-14T11:13:02,678 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_17180. 2024-04-14T11:13:02,678 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_19314. 2024-04-14T11:13:02,678 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_21406. 2024-04-14T11:13:02,678 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_23575. 2024-04-14T11:13:02,678 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_25763. 2024-04-14T11:13:02,679 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_27933. 2024-04-14T11:13:02,679 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_30029. 2024-04-14T11:13:02,679 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_32167. 2024-04-14T11:13:02,679 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_34326. 2024-04-14T11:13:02,680 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_36465. 2024-04-14T11:13:02,680 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_38607. 2024-04-14T11:13:02,680 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_40756. 2024-04-14T11:13:02,680 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_42950. 2024-04-14T11:13:02,681 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_45097. 2024-04-14T11:13:02,681 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_47212. 2024-04-14T11:13:02,681 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_49298. 2024-04-14T11:13:02,681 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_51466. 2024-04-14T11:13:02,682 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_53574. 2024-04-14T11:13:02,682 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_55710. 2024-04-14T11:13:02,682 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_57817. 2024-04-14T11:13:02,682 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_59962. 2024-04-14T11:13:02,683 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_62116. 2024-04-14T11:13:02,683 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_64299. 2024-04-14T11:13:02,683 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_66430. 2024-04-14T11:13:02,683 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_68577. 2024-04-14T11:13:02,684 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_70748. 2024-04-14T11:13:02,684 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_72877. 2024-04-14T11:13:02,684 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_75013. 2024-04-14T11:13:02,684 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_77110. 2024-04-14T11:13:02,684 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_79224. 2024-04-14T11:13:02,685 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_81416. 2024-04-14T11:13:02,685 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_83545. 2024-04-14T11:13:02,685 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_85676. 2024-04-14T11:13:02,685 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_87834. 2024-04-14T11:13:02,686 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_89934. 2024-04-14T11:13:02,686 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_92093. 2024-04-14T11:13:02,686 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_94212. 2024-04-14T11:13:02,686 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_96359. 2024-04-14T11:13:02,686 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_98453. 2024-04-14T11:13:02,687 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_100576. 2024-04-14T11:13:02,687 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_102675. 2024-04-14T11:13:02,687 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_104865. 2024-04-14T11:13:02,687 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_106999. 2024-04-14T11:13:02,687 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_109115. 2024-04-14T11:13:02,688 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_111273. 2024-04-14T11:13:02,688 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_113359. 2024-04-14T11:13:02,688 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_115495. 2024-04-14T11:13:02,688 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_117680. 2024-04-14T11:13:02,688 DEBUG [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key cf50821fedfd4f098e31b9522dcbb320_118051. 2024-04-14T11:13:02,689 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2064): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc221547-0085-fa7f-3168-33a7e0b8cd32/TestPrefetch1/cf50821fedfd4f098e31b9522dcbb320, with total blocks 57, and data blocks 56 2024-04-14T11:13:02,689 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2068): iterating over 70 entries in the backing map 2024-04-14T11:13:02,689 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_0 in the backing map. Acquiring read lock for offset 353280 2024-04-14T11:13:02,689 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_8674 in the backing map. Acquiring read lock for offset 334848 2024-04-14T11:13:02,689 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_4395 in the backing map. Acquiring read lock for offset 344064 2024-04-14T11:13:02,689 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_12856 in the backing map. Acquiring read lock for offset 328704 2024-04-14T11:13:02,689 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_21406 in the backing map. Acquiring read lock for offset 322560 2024-04-14T11:13:02,689 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_17180 in the backing map. Acquiring read lock for offset 347136 2024-04-14T11:13:02,689 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_2197 in the backing map. Acquiring read lock for offset 350208 2024-04-14T11:13:02,689 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_6559 in the backing map. Acquiring read lock for offset 340992 2024-04-14T11:13:02,689 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_15036 in the backing map. Acquiring read lock for offset 331776 2024-04-14T11:13:02,689 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_10764 in the backing map. Acquiring read lock for offset 337920 2024-04-14T11:13:02,689 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_19314 in the backing map. Acquiring read lock for offset 325632 2024-04-14T11:13:02,689 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_23575 in the backing map. Acquiring read lock for offset 316416 2024-04-14T11:13:02,690 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_32167 in the backing map. Acquiring read lock for offset 313344 2024-04-14T11:13:02,690 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_27933 in the backing map. Acquiring read lock for offset 319488 2024-04-14T11:13:02,690 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_36465 in the backing map. Acquiring read lock for offset 291840 2024-04-14T11:13:02,690 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_81416 in the backing map. Acquiring read lock for offset 279552 2024-04-14T11:13:02,690 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2096): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc221547-0085-fa7f-3168-33a7e0b8cd32/TestPrefetch1/cf50821fedfd4f098e31b9522dcbb320, but only 16 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue. 2024-04-14T11:13:02,690 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2105): There are still blocks pending caching for file cf50821fedfd4f098e31b9522dcbb320. Will sleep 100ms and try the verification again. 2024-04-14T11:13:02,693 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-04-14T11:13:02,693 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1310): Thread bucket-cache-persister started persisting bucket cache to file 2024-04-14T11:13:02,698 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1323): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-04-14T11:13:02,790 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2064): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc221547-0085-fa7f-3168-33a7e0b8cd32/TestPrefetch1/cf50821fedfd4f098e31b9522dcbb320, with total blocks 57, and data blocks 56 2024-04-14T11:13:02,790 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2068): iterating over 113 entries in the backing map 2024-04-14T11:13:02,790 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_0 in the backing map. Acquiring read lock for offset 353280 2024-04-14T11:13:02,791 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_89934 in the backing map. Acquiring read lock for offset 218112 2024-04-14T11:13:02,791 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_40756 in the backing map. Acquiring read lock for offset 297984 2024-04-14T11:13:02,791 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_98453 in the backing map. Acquiring read lock for offset 215040 2024-04-14T11:13:02,791 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_49298 in the backing map. Acquiring read lock for offset 294912 2024-04-14T11:13:02,791 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_45097 in the backing map. Acquiring read lock for offset 288768 2024-04-14T11:13:02,791 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_94212 in the backing map. Acquiring read lock for offset 208896 2024-04-14T11:13:02,791 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_106999 in the backing map. Acquiring read lock for offset 196608 2024-04-14T11:13:02,791 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_57817 in the backing map. Acquiring read lock for offset 264192 2024-04-14T11:13:02,791 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_8674 in the backing map. Acquiring read lock for offset 334848 2024-04-14T11:13:02,791 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_53574 in the backing map. Acquiring read lock for offset 267264 2024-04-14T11:13:02,791 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_4395 in the backing map. Acquiring read lock for offset 344064 2024-04-14T11:13:02,791 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_102675 in the backing map. Acquiring read lock for offset 199680 2024-04-14T11:13:02,791 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_111273 in the backing map. Acquiring read lock for offset 187392 2024-04-14T11:13:02,791 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_62116 in the backing map. Acquiring read lock for offset 248832 2024-04-14T11:13:02,791 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_12856 in the backing map. Acquiring read lock for offset 328704 2024-04-14T11:13:02,791 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_21406 in the backing map. Acquiring read lock for offset 322560 2024-04-14T11:13:02,791 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_66430 in the backing map. Acquiring read lock for offset 242688 2024-04-14T11:13:02,791 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_115495 in the backing map. Acquiring read lock for offset 184320 2024-04-14T11:13:02,791 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_17180 in the backing map. Acquiring read lock for offset 347136 2024-04-14T11:13:02,791 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_25763 in the backing map. Acquiring read lock for offset 310272 2024-04-14T11:13:02,791 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_70748 in the backing map. Acquiring read lock for offset 285696 2024-04-14T11:13:02,791 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_79224 in the backing map. Acquiring read lock for offset 245760 2024-04-14T11:13:02,792 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_30029 in the backing map. Acquiring read lock for offset 307200 2024-04-14T11:13:02,792 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_75013 in the backing map. Acquiring read lock for offset 233472 2024-04-14T11:13:02,792 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_38607 in the backing map. Acquiring read lock for offset 276480 2024-04-14T11:13:02,792 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_83545 in the backing map. Acquiring read lock for offset 227328 2024-04-14T11:13:02,792 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_34326 in the backing map. Acquiring read lock for offset 304128 2024-04-14T11:13:02,792 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_42950 in the backing map. Acquiring read lock for offset 301056 2024-04-14T11:13:02,792 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_92093 in the backing map. Acquiring read lock for offset 224256 2024-04-14T11:13:02,792 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_87834 in the backing map. Acquiring read lock for offset 239616 2024-04-14T11:13:02,792 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_100576 in the backing map. Acquiring read lock for offset 205824 2024-04-14T11:13:02,792 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_2197 in the backing map. Acquiring read lock for offset 350208 2024-04-14T11:13:02,792 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_96359 in the backing map. Acquiring read lock for offset 221184 2024-04-14T11:13:02,792 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_47212 in the backing map. Acquiring read lock for offset 282624 2024-04-14T11:13:02,792 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_104865 in the backing map. Acquiring read lock for offset 193536 2024-04-14T11:13:02,792 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_6559 in the backing map. Acquiring read lock for offset 340992 2024-04-14T11:13:02,792 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_55710 in the backing map. Acquiring read lock for offset 261120 2024-04-14T11:13:02,792 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_51466 in the backing map. Acquiring read lock for offset 273408 2024-04-14T11:13:02,792 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_113359 in the backing map. Acquiring read lock for offset 190464 2024-04-14T11:13:02,792 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_15036 in the backing map. Acquiring read lock for offset 331776 2024-04-14T11:13:02,792 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_109115 in the backing map. Acquiring read lock for offset 211968 2024-04-14T11:13:02,792 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_59962 in the backing map. Acquiring read lock for offset 254976 2024-04-14T11:13:02,792 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_10764 in the backing map. Acquiring read lock for offset 337920 2024-04-14T11:13:02,792 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_68577 in the backing map. Acquiring read lock for offset 251904 2024-04-14T11:13:02,792 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_117680 in the backing map. Acquiring read lock for offset 202752 2024-04-14T11:13:02,793 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_19314 in the backing map. Acquiring read lock for offset 325632 2024-04-14T11:13:02,793 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_64299 in the backing map. Acquiring read lock for offset 258048 2024-04-14T11:13:02,793 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_72877 in the backing map. Acquiring read lock for offset 236544 2024-04-14T11:13:02,793 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_23575 in the backing map. Acquiring read lock for offset 316416 2024-04-14T11:13:02,793 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_32167 in the backing map. Acquiring read lock for offset 313344 2024-04-14T11:13:02,793 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_77110 in the backing map. Acquiring read lock for offset 230400 2024-04-14T11:13:02,793 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_27933 in the backing map. Acquiring read lock for offset 319488 2024-04-14T11:13:02,793 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_85676 in the backing map. Acquiring read lock for offset 270336 2024-04-14T11:13:02,793 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_36465 in the backing map. Acquiring read lock for offset 291840 2024-04-14T11:13:02,793 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2075): found block cf50821fedfd4f098e31b9522dcbb320_81416 in the backing map. Acquiring read lock for offset 279552 2024-04-14T11:13:02,793 DEBUG [hfile-prefetch-1713093177644 {}] bucket.BucketCache(2093): File /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc221547-0085-fa7f-3168-33a7e0b8cd32/TestPrefetch1/cf50821fedfd4f098e31b9522dcbb320 has now been fully cached. 2024-04-14T11:13:02,793 WARN [hfile-prefetch-1713093177644 {}] hfile.HFilePreadReader$1(151): Prefetch path=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc221547-0085-fa7f-3168-33a7e0b8cd32/TestPrefetch1/cf50821fedfd4f098e31b9522dcbb320, offset=118344, end=118344 java.lang.IllegalMonitorStateException: attempt to unlock read lock, not locked by current thread at java.util.concurrent.locks.ReentrantReadWriteLock$Sync.unmatchedUnlockException(ReentrantReadWriteLock.java:444) ~[?:1.8.0_362] at java.util.concurrent.locks.ReentrantReadWriteLock$Sync.tryReleaseShared(ReentrantReadWriteLock.java:428) ~[?:1.8.0_362] at java.util.concurrent.locks.AbstractQueuedSynchronizer.releaseShared(AbstractQueuedSynchronizer.java:1341) ~[?:1.8.0_362] at java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.unlock(ReentrantReadWriteLock.java:881) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2125) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:141) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_362] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) ~[?:1.8.0_362] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-14T11:13:02,794 DEBUG [hfile-prefetch-1713093177644 {}] hfile.PrefetchExecutor(108): Prefetch completed for cf50821fedfd4f098e31b9522dcbb320 2024-04-14T11:13:06,726 INFO [Time-limited test {}] bucket.BucketCache(1534): Disabling cache 2024-04-14T11:13:06,727 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1094): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-04-14T11:13:06,727 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1094): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-04-14T11:13:06,727 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1094): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-04-14T11:13:06,727 INFO [Time-limited test {}] bucket.BucketCache(1558): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/cc221547-0085-fa7f-3168-33a7e0b8cd32/bucket.persistence 2024-04-14T11:13:06,728 INFO [Time-limited test {}] bucket.BucketCache(1564): Shutting down cache persister thread. 2024-04-14T11:13:06,728 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-04-14T11:13:06,738 DEBUG [Time-limited test {}] bucket.BucketCache(1310): Thread Time-limited test started persisting bucket cache to file 2024-04-14T11:13:06,743 DEBUG [Time-limited test {}] bucket.BucketCache(1323): Thread Time-limited test finished persisting bucket cache to file, renaming 2024-04-14T11:13:06,744 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-04-14T11:13:06,745 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-04-14T11:13:06,755 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestBucketCachePersister#testPrefetchPersistenceCrash Thread=17 (was 16) Potentially hanging thread: hfile-prefetch-1713093179721 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=312 (was 310) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=225 (was 236), ProcessCount=170 (was 170), AvailableMemoryMB=2847 (was 3129)