2024-04-12 03:12:39,027 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@f79e 2024-04-12 03:12:39,043 main DEBUG Took 0.013693 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-04-12 03:12:39,043 main DEBUG PluginManager 'Core' found 129 plugins 2024-04-12 03:12:39,044 main DEBUG PluginManager 'Level' found 0 plugins 2024-04-12 03:12:39,045 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-04-12 03:12:39,046 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-12 03:12:39,057 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-04-12 03:12:39,073 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-12 03:12:39,075 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-12 03:12:39,075 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-12 03:12:39,076 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-12 03:12:39,076 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-12 03:12:39,076 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-12 03:12:39,077 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-12 03:12:39,077 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-12 03:12:39,078 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-12 03:12:39,078 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-12 03:12:39,079 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-12 03:12:39,079 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-12 03:12:39,080 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-12 03:12:39,080 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-12 03:12:39,081 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-12 03:12:39,081 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-12 03:12:39,081 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-12 03:12:39,082 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-12 03:12:39,082 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-12 03:12:39,082 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-12 03:12:39,083 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-12 03:12:39,083 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-12 03:12:39,084 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-12 03:12:39,084 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-04-12 03:12:39,085 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-12 03:12:39,085 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-04-12 03:12:39,088 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-04-12 03:12:39,090 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-04-12 03:12:39,092 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-04-12 03:12:39,093 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-04-12 03:12:39,094 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-04-12 03:12:39,095 main DEBUG PluginManager 'Converter' found 47 plugins 2024-04-12 03:12:39,111 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-04-12 03:12:39,115 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-04-12 03:12:39,116 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-04-12 03:12:39,116 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-04-12 03:12:39,120 main DEBUG createAppenders(={Console}) 2024-04-12 03:12:39,121 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@f79e initialized 2024-04-12 03:12:39,121 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@f79e 2024-04-12 03:12:39,122 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@f79e OK. 2024-04-12 03:12:39,122 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-04-12 03:12:39,123 main DEBUG OutputStream closed 2024-04-12 03:12:39,123 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-04-12 03:12:39,123 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-04-12 03:12:39,124 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@6aeb35e6 OK 2024-04-12 03:12:39,181 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586 2024-04-12 03:12:39,183 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=StatusLogger 2024-04-12 03:12:39,184 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=ContextSelector 2024-04-12 03:12:39,185 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name= 2024-04-12 03:12:39,186 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.directory 2024-04-12 03:12:39,186 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-04-12 03:12:39,187 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.zookeeper 2024-04-12 03:12:39,187 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-04-12 03:12:39,188 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-04-12 03:12:39,188 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-04-12 03:12:39,188 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase 2024-04-12 03:12:39,189 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop 2024-04-12 03:12:39,189 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-04-12 03:12:39,189 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-04-12 03:12:39,190 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-04-12 03:12:39,190 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-04-12 03:12:39,191 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-04-12 03:12:39,192 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Appenders,name=Console 2024-04-12 03:12:39,195 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2024-04-12 03:12:39,196 main DEBUG Reconfiguration complete for context[name=1b6d3586] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.7.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@23c30a20) with optional ClassLoader: null 2024-04-12 03:12:39,196 main DEBUG Shutdown hook enabled. Registering a new one. 2024-04-12 03:12:39,197 main DEBUG LoggerContext[name=1b6d3586, org.apache.logging.log4j.core.LoggerContext@23c30a20] started OK. 2024-04-12T03:12:39,214 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.bucket.TestBucketCachePersister timeout: 13 mins 2024-04-12 03:12:39,219 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-04-12 03:12:39,219 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2024-04-12T03:12:39,523 DEBUG [main {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/699211b6-59f2-0bbd-30ca-c92e999a5a56 2024-04-12T03:12:39,560 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestBucketCachePersister#testPrefetchBlockEvictionWhilePrefetchRunning Thread=10, OpenFileDescriptor=306, MaxFileDescriptor=60000, SystemLoadAverage=304, ProcessCount=171, AvailableMemoryMB=5050 2024-04-12T03:12:39,713 WARN [Time-limited test {}] util.NativeCodeLoader(62): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-04-12T03:12:40,233 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-04-12T03:12:40,257 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/699211b6-59f2-0bbd-30ca-c92e999a5a56/testPrefetchBlockEvictionWhilePrefetchRunning 2024-04-12T03:12:40,258 INFO [Time-limited test {}] bucket.BucketCache(305): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2024-04-12T03:12:40,258 INFO [Time-limited test {}] bucket.BucketCache(393): Starting BucketCachePersisterThread 2024-04-12T03:12:40,259 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 200 2024-04-12T03:12:40,260 INFO [Time-limited test {}] bucket.BucketCache(1344): Started retrieving bucket cache from file 2024-04-12T03:12:40,260 WARN [Time-limited test {}] bucket.BucketCache(1347): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-04-12T03:12:40,273 INFO [Time-limited test {}] bucket.BucketAllocator(339): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-04-12T03:12:40,274 INFO [Time-limited test {}] bucket.BucketCache(356): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/699211b6-59f2-0bbd-30ca-c92e999a5a56/testPrefetchBlockEvictionWhilePrefetchRunning, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/699211b6-59f2-0bbd-30ca-c92e999a5a56/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-04-12T03:12:40,277 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-04-12T03:12:40,300 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-04-12T03:12:40,354 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 819b4805f5424615b537d31d2a715e72 2024-04-12T03:12:40,509 WARN [Time-limited test {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-04-12T03:12:40,553 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-04-12T03:12:40,600 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(813): Len of the biggest cell in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/699211b6-59f2-0bbd-30ca-c92e999a5a56/TestPrefetch3/819b4805f5424615b537d31d2a715e72 is 242, key is aaaaaaaaaaaaaaaaaaaaaabbaabbbbaa/lX/1418777406493573632/Delete/seqid=0 2024-04-12T03:12:40,755 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(92): Prefetch requested for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/699211b6-59f2-0bbd-30ca-c92e999a5a56/TestPrefetch3/819b4805f5424615b537d31d2a715e72, delay=967 ms 2024-04-12T03:12:41,734 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_0. 2024-04-12T03:12:41,813 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_2123. 2024-04-12T03:12:41,814 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_4309. 2024-04-12T03:12:41,815 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_6402. 2024-04-12T03:12:41,815 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_8505. 2024-04-12T03:12:41,816 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_10590. 2024-04-12T03:12:41,816 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_12680. 2024-04-12T03:12:41,817 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_14843. 2024-04-12T03:12:41,817 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_16983. 2024-04-12T03:12:41,818 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_19169. 2024-04-12T03:12:41,818 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_21270. 2024-04-12T03:12:41,818 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_23369. 2024-04-12T03:12:41,819 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_25470. 2024-04-12T03:12:41,819 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_27625. 2024-04-12T03:12:41,820 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_29734. 2024-04-12T03:12:41,820 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_31857. 2024-04-12T03:12:41,820 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_34039. 2024-04-12T03:12:41,821 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_36161. 2024-04-12T03:12:41,821 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_38271. 2024-04-12T03:12:41,822 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_40359. 2024-04-12T03:12:41,822 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_42575. 2024-04-12T03:12:41,822 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_44679. 2024-04-12T03:12:41,823 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_46826. 2024-04-12T03:12:41,823 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_48973. 2024-04-12T03:12:41,823 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_51059. 2024-04-12T03:12:41,824 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_53164. 2024-04-12T03:12:41,824 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_55303. 2024-04-12T03:12:41,824 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_57439. 2024-04-12T03:12:41,825 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_59592. 2024-04-12T03:12:41,825 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_61817. 2024-04-12T03:12:41,826 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_63949. 2024-04-12T03:12:41,826 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_66055. 2024-04-12T03:12:41,826 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_68179. 2024-04-12T03:12:41,827 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_70265. 2024-04-12T03:12:41,827 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_72352. 2024-04-12T03:12:41,827 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_74495. 2024-04-12T03:12:41,828 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_76629. 2024-04-12T03:12:41,828 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_78758. 2024-04-12T03:12:41,828 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_80890. 2024-04-12T03:12:41,828 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_83009. 2024-04-12T03:12:41,829 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_85162. 2024-04-12T03:12:41,829 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_87413. 2024-04-12T03:12:41,830 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_89540. 2024-04-12T03:12:41,830 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_91637. 2024-04-12T03:12:41,830 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_93774. 2024-04-12T03:12:41,831 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_95972. 2024-04-12T03:12:41,831 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_98214. 2024-04-12T03:12:41,831 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_100372. 2024-04-12T03:12:41,832 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_102491. 2024-04-12T03:12:41,832 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_104639. 2024-04-12T03:12:41,832 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_106786. 2024-04-12T03:12:41,833 DEBUG [Time-limited test {}] bucket.BucketCache(739): removed key 819b4805f5424615b537d31d2a715e72_10590 from back map with offset lock 488448 in the evict process 2024-04-12T03:12:41,833 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_108958. 2024-04-12T03:12:41,833 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_111076. 2024-04-12T03:12:41,834 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_113227. 2024-04-12T03:12:41,834 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_115333. 2024-04-12T03:12:41,834 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_117473. 2024-04-12T03:12:41,835 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 819b4805f5424615b537d31d2a715e72_118154. 2024-04-12T03:12:41,835 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2064): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/699211b6-59f2-0bbd-30ca-c92e999a5a56/TestPrefetch3/819b4805f5424615b537d31d2a715e72, with total blocks 57, and data blocks 56 2024-04-12T03:12:41,836 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2068): iterating over 31 entries in the backing map 2024-04-12T03:12:41,837 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block 819b4805f5424615b537d31d2a715e72_6402 in the backing map. Acquiring read lock for offset 513024 2024-04-12T03:12:41,838 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block 819b4805f5424615b537d31d2a715e72_55303 in the backing map. Acquiring read lock for offset 463872 2024-04-12T03:12:41,838 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block 819b4805f5424615b537d31d2a715e72_59592 in the backing map. Acquiring read lock for offset 433152 2024-04-12T03:12:41,838 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block 819b4805f5424615b537d31d2a715e72_2123 in the backing map. Acquiring read lock for offset 525312 2024-04-12T03:12:41,838 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block 819b4805f5424615b537d31d2a715e72_46826 in the backing map. Acquiring read lock for offset 451584 2024-04-12T03:12:41,838 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block 819b4805f5424615b537d31d2a715e72_51059 in the backing map. Acquiring read lock for offset 448512 2024-04-12T03:12:41,839 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block 819b4805f5424615b537d31d2a715e72_42575 in the backing map. Acquiring read lock for offset 454656 2024-04-12T03:12:41,839 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block 819b4805f5424615b537d31d2a715e72_36161 in the backing map. Acquiring read lock for offset 485376 2024-04-12T03:12:41,839 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block 819b4805f5424615b537d31d2a715e72_89540 in the backing map. Acquiring read lock for offset 402432 2024-04-12T03:12:41,839 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block 819b4805f5424615b537d31d2a715e72_40359 in the backing map. Acquiring read lock for offset 439296 2024-04-12T03:12:41,839 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block 819b4805f5424615b537d31d2a715e72_80890 in the backing map. Acquiring read lock for offset 399360 2024-04-12T03:12:41,839 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block 819b4805f5424615b537d31d2a715e72_27625 in the backing map. Acquiring read lock for offset 509952 2024-04-12T03:12:41,839 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block 819b4805f5424615b537d31d2a715e72_85162 in the backing map. Acquiring read lock for offset 393216 2024-04-12T03:12:41,840 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block 819b4805f5424615b537d31d2a715e72_31857 in the backing map. Acquiring read lock for offset 473088 2024-04-12T03:12:41,840 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block 819b4805f5424615b537d31d2a715e72_23369 in the backing map. Acquiring read lock for offset 491520 2024-04-12T03:12:41,840 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block 819b4805f5424615b537d31d2a715e72_19169 in the backing map. Acquiring read lock for offset 497664 2024-04-12T03:12:41,840 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block 819b4805f5424615b537d31d2a715e72_76629 in the backing map. Acquiring read lock for offset 430080 2024-04-12T03:12:41,840 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block 819b4805f5424615b537d31d2a715e72_14843 in the backing map. Acquiring read lock for offset 506880 2024-04-12T03:12:41,840 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block 819b4805f5424615b537d31d2a715e72_63949 in the backing map. Acquiring read lock for offset 457728 2024-04-12T03:12:41,841 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block 819b4805f5424615b537d31d2a715e72_72352 in the backing map. Acquiring read lock for offset 405504 2024-04-12T03:12:41,841 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block 819b4805f5424615b537d31d2a715e72_68179 in the backing map. Acquiring read lock for offset 411648 2024-04-12T03:12:41,841 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block 819b4805f5424615b537d31d2a715e72_8505 in the backing map. Acquiring read lock for offset 500736 2024-04-12T03:12:41,841 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block 819b4805f5424615b537d31d2a715e72_4309 in the backing map. Acquiring read lock for offset 519168 2024-04-12T03:12:41,841 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block 819b4805f5424615b537d31d2a715e72_12680 in the backing map. Acquiring read lock for offset 516096 2024-04-12T03:12:41,841 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block 819b4805f5424615b537d31d2a715e72_61817 in the backing map. Acquiring read lock for offset 445440 2024-04-12T03:12:41,841 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block 819b4805f5424615b537d31d2a715e72_0 in the backing map. Acquiring read lock for offset 522240 2024-04-12T03:12:41,842 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block 819b4805f5424615b537d31d2a715e72_57439 in the backing map. Acquiring read lock for offset 460800 2024-04-12T03:12:41,842 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block 819b4805f5424615b537d31d2a715e72_102491 in the backing map. Acquiring read lock for offset 387072 2024-04-12T03:12:41,842 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block 819b4805f5424615b537d31d2a715e72_53164 in the backing map. Acquiring read lock for offset 466944 2024-04-12T03:12:41,842 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block 819b4805f5424615b537d31d2a715e72_98214 in the backing map. Acquiring read lock for offset 377856 2024-04-12T03:12:41,842 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block 819b4805f5424615b537d31d2a715e72_48973 in the backing map. Acquiring read lock for offset 470016 2024-04-12T03:12:41,842 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block 819b4805f5424615b537d31d2a715e72_44679 in the backing map. Acquiring read lock for offset 436224 2024-04-12T03:12:41,842 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block 819b4805f5424615b537d31d2a715e72_93774 in the backing map. Acquiring read lock for offset 380928 2024-04-12T03:12:41,843 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block 819b4805f5424615b537d31d2a715e72_34039 in the backing map. Acquiring read lock for offset 494592 2024-04-12T03:12:41,843 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block 819b4805f5424615b537d31d2a715e72_38271 in the backing map. Acquiring read lock for offset 442368 2024-04-12T03:12:41,843 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block 819b4805f5424615b537d31d2a715e72_87413 in the backing map. Acquiring read lock for offset 408576 2024-04-12T03:12:41,843 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block 819b4805f5424615b537d31d2a715e72_83009 in the backing map. Acquiring read lock for offset 414720 2024-04-12T03:12:41,843 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block 819b4805f5424615b537d31d2a715e72_29734 in the backing map. Acquiring read lock for offset 503808 2024-04-12T03:12:41,843 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block 819b4805f5424615b537d31d2a715e72_21270 in the backing map. Acquiring read lock for offset 479232 2024-04-12T03:12:41,843 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block 819b4805f5424615b537d31d2a715e72_74495 in the backing map. Acquiring read lock for offset 423936 2024-04-12T03:12:41,843 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block 819b4805f5424615b537d31d2a715e72_78758 in the backing map. Acquiring read lock for offset 417792 2024-04-12T03:12:41,844 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block 819b4805f5424615b537d31d2a715e72_25470 in the backing map. Acquiring read lock for offset 476160 2024-04-12T03:12:41,844 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block 819b4805f5424615b537d31d2a715e72_66055 in the backing map. Acquiring read lock for offset 420864 2024-04-12T03:12:41,844 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block 819b4805f5424615b537d31d2a715e72_70265 in the backing map. Acquiring read lock for offset 427008 2024-04-12T03:12:41,844 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block 819b4805f5424615b537d31d2a715e72_16983 in the backing map. Acquiring read lock for offset 482304 2024-04-12T03:12:41,844 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2096): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/699211b6-59f2-0bbd-30ca-c92e999a5a56/TestPrefetch3/819b4805f5424615b537d31d2a715e72, but only 45 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue. 2024-04-12T03:12:41,847 INFO [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2116): We found only 45 data blocks cached from a total of 56 for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/699211b6-59f2-0bbd-30ca-c92e999a5a56/TestPrefetch3/819b4805f5424615b537d31d2a715e72, but no blocks pending caching. Maybe cache is full or evictions happened concurrently to cache prefetch. 2024-04-12T03:12:41,849 DEBUG [hfile-prefetch-1712891560758 {}] hfile.PrefetchExecutor(108): Prefetch completed for 819b4805f5424615b537d31d2a715e72 2024-04-12T03:12:41,853 INFO [Time-limited test {}] bucket.BucketCache(1534): Disabling cache 2024-04-12T03:12:41,854 INFO [Time-limited test {}] bucket.BucketCache(1558): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/699211b6-59f2-0bbd-30ca-c92e999a5a56/bucket.persistence 2024-04-12T03:12:41,854 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1094): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-04-12T03:12:41,854 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1094): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-04-12T03:12:41,854 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1094): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-04-12T03:12:41,855 INFO [Time-limited test {}] bucket.BucketCache(1564): Shutting down cache persister thread. 2024-04-12T03:12:41,855 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-04-12T03:12:41,865 DEBUG [Time-limited test {}] bucket.BucketCache(1310): Thread Time-limited test started persisting bucket cache to file 2024-04-12T03:12:41,990 DEBUG [Time-limited test {}] bucket.BucketCache(1323): Thread Time-limited test finished persisting bucket cache to file, renaming 2024-04-12T03:12:41,991 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-04-12T03:12:41,992 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-04-12T03:12:41,993 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-04-12T03:12:42,004 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestBucketCachePersister#testPrefetchBlockEvictionWhilePrefetchRunning Thread=14 (was 10) Potentially hanging thread: hfile-prefetch-1712891560758 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Timer for 'HBase' metrics system java.lang.Object.wait(Native Method) java.util.TimerThread.mainLoop(Timer.java:552) java.util.TimerThread.run(Timer.java:505) Potentially hanging thread: HBase-Metrics2-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.lang.Object.wait(Native Method) java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:144) java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:165) org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:3712) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=308 (was 306) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=304 (was 304), ProcessCount=171 (was 171), AvailableMemoryMB=4875 (was 5050) 2024-04-12T03:12:42,014 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestBucketCachePersister#testPrefetchPersistenceCrashNegative Thread=14, OpenFileDescriptor=308, MaxFileDescriptor=60000, SystemLoadAverage=304, ProcessCount=171, AvailableMemoryMB=4874 2024-04-12T03:12:42,014 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-04-12T03:12:42,015 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/699211b6-59f2-0bbd-30ca-c92e999a5a56/testPrefetchPersistenceCrashNegative 2024-04-12T03:12:42,015 INFO [Time-limited test {}] bucket.BucketCache(305): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2024-04-12T03:12:42,015 INFO [Time-limited test {}] bucket.BucketCache(393): Starting BucketCachePersisterThread 2024-04-12T03:12:42,015 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 9223372036854775807 2024-04-12T03:12:42,015 INFO [Time-limited test {}] bucket.BucketCache(1344): Started retrieving bucket cache from file 2024-04-12T03:12:42,016 WARN [Time-limited test {}] bucket.BucketCache(1347): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-04-12T03:12:42,016 INFO [Time-limited test {}] bucket.BucketAllocator(339): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-04-12T03:12:42,018 INFO [Time-limited test {}] bucket.BucketCache(356): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/699211b6-59f2-0bbd-30ca-c92e999a5a56/testPrefetchPersistenceCrashNegative, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/699211b6-59f2-0bbd-30ca-c92e999a5a56/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-04-12T03:12:42,018 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-04-12T03:12:42,022 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f81bd56599624288a21e83083bb32441 2024-04-12T03:12:42,036 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(813): Len of the biggest cell in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/699211b6-59f2-0bbd-30ca-c92e999a5a56/TestPrefetch2/f81bd56599624288a21e83083bb32441 is 243, key is aaaaaaaaaaaaaaaaaaaaaabaababaaba/xu/1017465087028669020/DeleteFamily/seqid=0 2024-04-12T03:12:42,038 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(92): Prefetch requested for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/699211b6-59f2-0bbd-30ca-c92e999a5a56/TestPrefetch2/f81bd56599624288a21e83083bb32441, delay=901 ms 2024-04-12T03:12:42,940 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_0. 2024-04-12T03:12:42,941 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_2108. 2024-04-12T03:12:42,941 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_4201. 2024-04-12T03:12:42,942 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_6427. 2024-04-12T03:12:42,942 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_8596. 2024-04-12T03:12:42,942 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_10741. 2024-04-12T03:12:42,942 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_12828. 2024-04-12T03:12:42,943 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_14916. 2024-04-12T03:12:42,943 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_17087. 2024-04-12T03:12:42,943 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_19247. 2024-04-12T03:12:42,943 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_21423. 2024-04-12T03:12:42,944 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_23596. 2024-04-12T03:12:42,944 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_25830. 2024-04-12T03:12:42,944 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_27922. 2024-04-12T03:12:42,944 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_30060. 2024-04-12T03:12:42,945 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_32213. 2024-04-12T03:12:42,945 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_34378. 2024-04-12T03:12:42,945 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_36511. 2024-04-12T03:12:42,946 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_38611. 2024-04-12T03:12:42,946 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_40763. 2024-04-12T03:12:42,946 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_42898. 2024-04-12T03:12:42,946 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_44991. 2024-04-12T03:12:42,947 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_47190. 2024-04-12T03:12:42,947 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_49356. 2024-04-12T03:12:42,947 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_51482. 2024-04-12T03:12:42,947 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_53627. 2024-04-12T03:12:42,948 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_55784. 2024-04-12T03:12:42,948 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_57892. 2024-04-12T03:12:42,949 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_59993. 2024-04-12T03:12:42,949 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_62097. 2024-04-12T03:12:42,949 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_64257. 2024-04-12T03:12:42,949 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_66389. 2024-04-12T03:12:42,950 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_68553. 2024-04-12T03:12:42,950 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_70729. 2024-04-12T03:12:42,950 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_72903. 2024-04-12T03:12:42,950 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_75085. 2024-04-12T03:12:42,950 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_77172. 2024-04-12T03:12:42,951 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_79320. 2024-04-12T03:12:42,951 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_81508. 2024-04-12T03:12:42,951 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_83683. 2024-04-12T03:12:42,952 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_85821. 2024-04-12T03:12:42,952 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_88009. 2024-04-12T03:12:42,952 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_90154. 2024-04-12T03:12:42,953 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_92309. 2024-04-12T03:12:42,953 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_94442. 2024-04-12T03:12:42,953 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_96563. 2024-04-12T03:12:42,953 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_98708. 2024-04-12T03:12:42,953 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_100833. 2024-04-12T03:12:42,954 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_102921. 2024-04-12T03:12:42,954 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_105042. 2024-04-12T03:12:42,954 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_107212. 2024-04-12T03:12:42,954 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_109438. 2024-04-12T03:12:42,955 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_111528. 2024-04-12T03:12:42,955 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_113640. 2024-04-12T03:12:42,955 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_115742. 2024-04-12T03:12:42,955 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key f81bd56599624288a21e83083bb32441_117437. 2024-04-12T03:12:42,956 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2064): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/699211b6-59f2-0bbd-30ca-c92e999a5a56/TestPrefetch2/f81bd56599624288a21e83083bb32441, with total blocks 56, and data blocks 55 2024-04-12T03:12:42,956 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2068): iterating over 11 entries in the backing map 2024-04-12T03:12:42,956 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_30060 in the backing map. Acquiring read lock for offset 494592 2024-04-12T03:12:42,956 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_25830 in the backing map. Acquiring read lock for offset 497664 2024-04-12T03:12:42,956 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_21423 in the backing map. Acquiring read lock for offset 503808 2024-04-12T03:12:42,956 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_4201 in the backing map. Acquiring read lock for offset 519168 2024-04-12T03:12:42,956 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_0 in the backing map. Acquiring read lock for offset 525312 2024-04-12T03:12:42,957 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_17087 in the backing map. Acquiring read lock for offset 516096 2024-04-12T03:12:42,957 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_12828 in the backing map. Acquiring read lock for offset 513024 2024-04-12T03:12:42,957 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_8596 in the backing map. Acquiring read lock for offset 488448 2024-04-12T03:12:42,957 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_40763 in the backing map. Acquiring read lock for offset 463872 2024-04-12T03:12:42,958 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_36511 in the backing map. Acquiring read lock for offset 473088 2024-04-12T03:12:42,958 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_32213 in the backing map. Acquiring read lock for offset 500736 2024-04-12T03:12:42,959 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_6427 in the backing map. Acquiring read lock for offset 491520 2024-04-12T03:12:42,959 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_51482 in the backing map. Acquiring read lock for offset 460800 2024-04-12T03:12:42,959 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_2108 in the backing map. Acquiring read lock for offset 522240 2024-04-12T03:12:42,959 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_42898 in the backing map. Acquiring read lock for offset 466944 2024-04-12T03:12:42,959 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_19247 in the backing map. Acquiring read lock for offset 506880 2024-04-12T03:12:42,959 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_14916 in the backing map. Acquiring read lock for offset 509952 2024-04-12T03:12:42,959 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_59993 in the backing map. Acquiring read lock for offset 451584 2024-04-12T03:12:42,959 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_10741 in the backing map. Acquiring read lock for offset 485376 2024-04-12T03:12:42,959 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_55784 in the backing map. Acquiring read lock for offset 454656 2024-04-12T03:12:42,960 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2096): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/699211b6-59f2-0bbd-30ca-c92e999a5a56/TestPrefetch2/f81bd56599624288a21e83083bb32441, but only 20 data blocks were cached. Total data blocks for file: 55. Checking for blocks pending cache in cache writer queue. 2024-04-12T03:12:42,960 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2105): There are still blocks pending caching for file f81bd56599624288a21e83083bb32441. Will sleep 100ms and try the verification again. 2024-04-12T03:12:43,060 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2064): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/699211b6-59f2-0bbd-30ca-c92e999a5a56/TestPrefetch2/f81bd56599624288a21e83083bb32441, with total blocks 56, and data blocks 55 2024-04-12T03:12:43,060 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2068): iterating over 56 entries in the backing map 2024-04-12T03:12:43,060 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_75085 in the backing map. Acquiring read lock for offset 380928 2024-04-12T03:12:43,061 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_30060 in the backing map. Acquiring read lock for offset 494592 2024-04-12T03:12:43,061 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_25830 in the backing map. Acquiring read lock for offset 497664 2024-04-12T03:12:43,061 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_70729 in the backing map. Acquiring read lock for offset 384000 2024-04-12T03:12:43,061 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_115742 in the backing map. Acquiring read lock for offset 390144 2024-04-12T03:12:43,061 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_21423 in the backing map. Acquiring read lock for offset 503808 2024-04-12T03:12:43,062 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_111528 in the backing map. Acquiring read lock for offset 359424 2024-04-12T03:12:43,062 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_38611 in the backing map. Acquiring read lock for offset 470016 2024-04-12T03:12:43,062 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_83683 in the backing map. Acquiring read lock for offset 374784 2024-04-12T03:12:43,062 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_34378 in the backing map. Acquiring read lock for offset 476160 2024-04-12T03:12:43,062 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_79320 in the backing map. Acquiring read lock for offset 420864 2024-04-12T03:12:43,062 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_49356 in the backing map. Acquiring read lock for offset 439296 2024-04-12T03:12:43,063 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_94442 in the backing map. Acquiring read lock for offset 402432 2024-04-12T03:12:43,063 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_4201 in the backing map. Acquiring read lock for offset 519168 2024-04-12T03:12:43,063 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_0 in the backing map. Acquiring read lock for offset 525312 2024-04-12T03:12:43,063 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_90154 in the backing map. Acquiring read lock for offset 368640 2024-04-12T03:12:43,063 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_44991 in the backing map. Acquiring read lock for offset 445440 2024-04-12T03:12:43,063 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_66389 in the backing map. Acquiring read lock for offset 448512 2024-04-12T03:12:43,063 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_107212 in the backing map. Acquiring read lock for offset 365568 2024-04-12T03:12:43,063 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_17087 in the backing map. Acquiring read lock for offset 516096 2024-04-12T03:12:43,064 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_62097 in the backing map. Acquiring read lock for offset 436224 2024-04-12T03:12:43,064 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_102921 in the backing map. Acquiring read lock for offset 411648 2024-04-12T03:12:43,064 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_57892 in the backing map. Acquiring read lock for offset 393216 2024-04-12T03:12:43,064 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_12828 in the backing map. Acquiring read lock for offset 513024 2024-04-12T03:12:43,064 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_53627 in the backing map. Acquiring read lock for offset 457728 2024-04-12T03:12:43,065 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_8596 in the backing map. Acquiring read lock for offset 488448 2024-04-12T03:12:43,065 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_98708 in the backing map. Acquiring read lock for offset 396288 2024-04-12T03:12:43,065 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_27922 in the backing map. Acquiring read lock for offset 479232 2024-04-12T03:12:43,065 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_72903 in the backing map. Acquiring read lock for offset 427008 2024-04-12T03:12:43,065 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_23596 in the backing map. Acquiring read lock for offset 482304 2024-04-12T03:12:43,065 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_68553 in the backing map. Acquiring read lock for offset 430080 2024-04-12T03:12:43,065 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_113640 in the backing map. Acquiring read lock for offset 356352 2024-04-12T03:12:43,066 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_109438 in the backing map. Acquiring read lock for offset 362496 2024-04-12T03:12:43,066 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_85821 in the backing map. Acquiring read lock for offset 417792 2024-04-12T03:12:43,066 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_40763 in the backing map. Acquiring read lock for offset 463872 2024-04-12T03:12:43,066 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_36511 in the backing map. Acquiring read lock for offset 473088 2024-04-12T03:12:43,066 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_81508 in the backing map. Acquiring read lock for offset 377856 2024-04-12T03:12:43,066 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_32213 in the backing map. Acquiring read lock for offset 500736 2024-04-12T03:12:43,067 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_77172 in the backing map. Acquiring read lock for offset 423936 2024-04-12T03:12:43,067 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_96563 in the backing map. Acquiring read lock for offset 399360 2024-04-12T03:12:43,067 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_6427 in the backing map. Acquiring read lock for offset 491520 2024-04-12T03:12:43,067 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_51482 in the backing map. Acquiring read lock for offset 460800 2024-04-12T03:12:43,067 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_92309 in the backing map. Acquiring read lock for offset 405504 2024-04-12T03:12:43,067 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_2108 in the backing map. Acquiring read lock for offset 522240 2024-04-12T03:12:43,067 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_47190 in the backing map. Acquiring read lock for offset 442368 2024-04-12T03:12:43,067 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_88009 in the backing map. Acquiring read lock for offset 371712 2024-04-12T03:12:43,068 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_42898 in the backing map. Acquiring read lock for offset 466944 2024-04-12T03:12:43,068 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_64257 in the backing map. Acquiring read lock for offset 433152 2024-04-12T03:12:43,068 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_19247 in the backing map. Acquiring read lock for offset 506880 2024-04-12T03:12:43,068 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_105042 in the backing map. Acquiring read lock for offset 408576 2024-04-12T03:12:43,068 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_14916 in the backing map. Acquiring read lock for offset 509952 2024-04-12T03:12:43,068 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_59993 in the backing map. Acquiring read lock for offset 451584 2024-04-12T03:12:43,069 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_10741 in the backing map. Acquiring read lock for offset 485376 2024-04-12T03:12:43,069 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_55784 in the backing map. Acquiring read lock for offset 454656 2024-04-12T03:12:43,069 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block f81bd56599624288a21e83083bb32441_100833 in the backing map. Acquiring read lock for offset 414720 2024-04-12T03:12:43,069 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2093): File /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/699211b6-59f2-0bbd-30ca-c92e999a5a56/TestPrefetch2/f81bd56599624288a21e83083bb32441 has now been fully cached. 2024-04-12T03:12:43,070 WARN [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(151): Prefetch path=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/699211b6-59f2-0bbd-30ca-c92e999a5a56/TestPrefetch2/f81bd56599624288a21e83083bb32441, offset=117730, end=117730 java.lang.IllegalMonitorStateException: attempt to unlock read lock, not locked by current thread at java.util.concurrent.locks.ReentrantReadWriteLock$Sync.unmatchedUnlockException(ReentrantReadWriteLock.java:444) ~[?:1.8.0_362] at java.util.concurrent.locks.ReentrantReadWriteLock$Sync.tryReleaseShared(ReentrantReadWriteLock.java:428) ~[?:1.8.0_362] at java.util.concurrent.locks.AbstractQueuedSynchronizer.releaseShared(AbstractQueuedSynchronizer.java:1341) ~[?:1.8.0_362] at java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.unlock(ReentrantReadWriteLock.java:881) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2125) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:141) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_362] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) ~[?:1.8.0_362] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-12T03:12:43,070 DEBUG [hfile-prefetch-1712891560758 {}] hfile.PrefetchExecutor(108): Prefetch completed for f81bd56599624288a21e83083bb32441 2024-04-12T03:12:44,042 INFO [Time-limited test {}] bucket.BucketCache(1534): Disabling cache 2024-04-12T03:12:44,043 INFO [Time-limited test {}] bucket.BucketCache(1558): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/699211b6-59f2-0bbd-30ca-c92e999a5a56/bucket.persistence 2024-04-12T03:12:44,043 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1094): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-04-12T03:12:44,043 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1094): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-04-12T03:12:44,043 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1094): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-04-12T03:12:44,043 INFO [Time-limited test {}] bucket.BucketCache(1564): Shutting down cache persister thread. 2024-04-12T03:12:44,043 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-04-12T03:12:44,054 DEBUG [Time-limited test {}] bucket.BucketCache(1310): Thread Time-limited test started persisting bucket cache to file 2024-04-12T03:12:44,061 DEBUG [Time-limited test {}] bucket.BucketCache(1323): Thread Time-limited test finished persisting bucket cache to file, renaming 2024-04-12T03:12:44,062 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-04-12T03:12:44,063 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-04-12T03:12:44,074 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestBucketCachePersister#testPrefetchPersistenceCrashNegative Thread=15 (was 14) Potentially hanging thread: hfile-prefetch-1712891562038 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=309 (was 308) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=288 (was 304), ProcessCount=171 (was 171), AvailableMemoryMB=4878 (was 4874) - AvailableMemoryMB LEAK? - 2024-04-12T03:12:44,091 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestBucketCachePersister#testPrefetchListUponBlockEviction Thread=15, OpenFileDescriptor=309, MaxFileDescriptor=60000, SystemLoadAverage=288, ProcessCount=171, AvailableMemoryMB=4877 2024-04-12T03:12:44,092 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-04-12T03:12:44,092 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/699211b6-59f2-0bbd-30ca-c92e999a5a56/testPrefetchListUponBlockEviction 2024-04-12T03:12:44,093 INFO [Time-limited test {}] bucket.BucketCache(305): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2024-04-12T03:12:44,093 INFO [Time-limited test {}] bucket.BucketCache(393): Starting BucketCachePersisterThread 2024-04-12T03:12:44,093 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 200 2024-04-12T03:12:44,093 INFO [Time-limited test {}] bucket.BucketCache(1344): Started retrieving bucket cache from file 2024-04-12T03:12:44,093 WARN [Time-limited test {}] bucket.BucketCache(1347): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-04-12T03:12:44,094 INFO [Time-limited test {}] bucket.BucketAllocator(339): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-04-12T03:12:44,096 INFO [Time-limited test {}] bucket.BucketCache(356): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/699211b6-59f2-0bbd-30ca-c92e999a5a56/testPrefetchListUponBlockEviction, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/699211b6-59f2-0bbd-30ca-c92e999a5a56/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-04-12T03:12:44,097 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-04-12T03:12:44,101 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 5347477213354343993cd2ff506bfcba 2024-04-12T03:12:44,111 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(813): Len of the biggest cell in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/699211b6-59f2-0bbd-30ca-c92e999a5a56/TestPrefetch3/5347477213354343993cd2ff506bfcba is 230, key is aaaaaaaaaaaaaaaaaaaaaaabaaabbabb/CGC:h0QVo/4304766506854242409/Put/seqid=0 2024-04-12T03:12:44,112 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(92): Prefetch requested for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/699211b6-59f2-0bbd-30ca-c92e999a5a56/TestPrefetch3/5347477213354343993cd2ff506bfcba, delay=996 ms 2024-04-12T03:12:44,293 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-04-12T03:12:44,294 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1310): Thread bucket-cache-persister started persisting bucket cache to file 2024-04-12T03:12:44,297 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1323): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-04-12T03:12:45,110 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_0. 2024-04-12T03:12:45,111 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_2104. 2024-04-12T03:12:45,111 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_4198. 2024-04-12T03:12:45,112 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_6368. 2024-04-12T03:12:45,112 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_8568. 2024-04-12T03:12:45,112 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_10696. 2024-04-12T03:12:45,113 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_12844. 2024-04-12T03:12:45,113 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_15046. 2024-04-12T03:12:45,113 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_17132. 2024-04-12T03:12:45,114 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_19281. 2024-04-12T03:12:45,114 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_21414. 2024-04-12T03:12:45,114 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_23618. 2024-04-12T03:12:45,114 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_25874. 2024-04-12T03:12:45,115 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_28040. 2024-04-12T03:12:45,115 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_30210. 2024-04-12T03:12:45,115 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_32339. 2024-04-12T03:12:45,116 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_34469. 2024-04-12T03:12:45,116 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_36631. 2024-04-12T03:12:45,116 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_38723. 2024-04-12T03:12:45,117 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_40813. 2024-04-12T03:12:45,117 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_42984. 2024-04-12T03:12:45,117 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_45110. 2024-04-12T03:12:45,117 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_47217. 2024-04-12T03:12:45,118 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_49429. 2024-04-12T03:12:45,118 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_51541. 2024-04-12T03:12:45,118 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_53682. 2024-04-12T03:12:45,119 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_55894. 2024-04-12T03:12:45,119 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_57985. 2024-04-12T03:12:45,119 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_60134. 2024-04-12T03:12:45,119 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_62250. 2024-04-12T03:12:45,120 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_64347. 2024-04-12T03:12:45,120 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_66480. 2024-04-12T03:12:45,120 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_68693. 2024-04-12T03:12:45,121 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_70796. 2024-04-12T03:12:45,121 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_72909. 2024-04-12T03:12:45,121 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_75056. 2024-04-12T03:12:45,121 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_77163. 2024-04-12T03:12:45,122 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_79271. 2024-04-12T03:12:45,122 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_81436. 2024-04-12T03:12:45,122 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_83554. 2024-04-12T03:12:45,122 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_85668. 2024-04-12T03:12:45,123 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_87782. 2024-04-12T03:12:45,123 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_89920. 2024-04-12T03:12:45,123 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_92052. 2024-04-12T03:12:45,124 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_94203. 2024-04-12T03:12:45,124 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_96396. 2024-04-12T03:12:45,124 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_98515. 2024-04-12T03:12:45,124 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_100709. 2024-04-12T03:12:45,125 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_102857. 2024-04-12T03:12:45,125 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_105059. 2024-04-12T03:12:45,125 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_107217. 2024-04-12T03:12:45,125 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_109384. 2024-04-12T03:12:45,126 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_111573. 2024-04-12T03:12:45,126 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_113758. 2024-04-12T03:12:45,126 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_115904. 2024-04-12T03:12:45,126 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_118046. 2024-04-12T03:12:45,127 DEBUG [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key 5347477213354343993cd2ff506bfcba_119775. 2024-04-12T03:12:45,127 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2064): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/699211b6-59f2-0bbd-30ca-c92e999a5a56/TestPrefetch3/5347477213354343993cd2ff506bfcba, with total blocks 57, and data blocks 56 2024-04-12T03:12:45,127 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2068): iterating over 3 entries in the backing map 2024-04-12T03:12:45,127 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_0 in the backing map. Acquiring read lock for offset 525312 2024-04-12T03:12:45,127 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_2104 in the backing map. Acquiring read lock for offset 522240 2024-04-12T03:12:45,127 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_6368 in the backing map. Acquiring read lock for offset 519168 2024-04-12T03:12:45,127 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2096): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/699211b6-59f2-0bbd-30ca-c92e999a5a56/TestPrefetch3/5347477213354343993cd2ff506bfcba, but only 3 data blocks were cached. Total data blocks for file: 56. Checking for blocks pending cache in cache writer queue. 2024-04-12T03:12:45,128 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2105): There are still blocks pending caching for file 5347477213354343993cd2ff506bfcba. Will sleep 100ms and try the verification again. 2024-04-12T03:12:45,228 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2064): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/699211b6-59f2-0bbd-30ca-c92e999a5a56/TestPrefetch3/5347477213354343993cd2ff506bfcba, with total blocks 57, and data blocks 56 2024-04-12T03:12:45,228 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2068): iterating over 57 entries in the backing map 2024-04-12T03:12:45,228 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_40813 in the backing map. Acquiring read lock for offset 479232 2024-04-12T03:12:45,228 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_0 in the backing map. Acquiring read lock for offset 525312 2024-04-12T03:12:45,228 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_94203 in the backing map. Acquiring read lock for offset 371712 2024-04-12T03:12:45,228 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_45110 in the backing map. Acquiring read lock for offset 439296 2024-04-12T03:12:45,228 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_4198 in the backing map. Acquiring read lock for offset 513024 2024-04-12T03:12:45,228 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_98515 in the backing map. Acquiring read lock for offset 387072 2024-04-12T03:12:45,228 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_49429 in the backing map. Acquiring read lock for offset 476160 2024-04-12T03:12:45,229 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_77163 in the backing map. Acquiring read lock for offset 390144 2024-04-12T03:12:45,229 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_28040 in the backing map. Acquiring read lock for offset 451584 2024-04-12T03:12:45,229 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_81436 in the backing map. Acquiring read lock for offset 402432 2024-04-12T03:12:45,229 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_32339 in the backing map. Acquiring read lock for offset 448512 2024-04-12T03:12:45,229 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_85668 in the backing map. Acquiring read lock for offset 384000 2024-04-12T03:12:45,229 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_89920 in the backing map. Acquiring read lock for offset 393216 2024-04-12T03:12:45,229 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_36631 in the backing map. Acquiring read lock for offset 485376 2024-04-12T03:12:45,229 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_109384 in the backing map. Acquiring read lock for offset 380928 2024-04-12T03:12:45,229 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_19281 in the backing map. Acquiring read lock for offset 509952 2024-04-12T03:12:45,229 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_64347 in the backing map. Acquiring read lock for offset 408576 2024-04-12T03:12:45,229 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_23618 in the backing map. Acquiring read lock for offset 457728 2024-04-12T03:12:45,229 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_68693 in the backing map. Acquiring read lock for offset 433152 2024-04-12T03:12:45,229 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_113758 in the backing map. Acquiring read lock for offset 377856 2024-04-12T03:12:45,229 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_72909 in the backing map. Acquiring read lock for offset 399360 2024-04-12T03:12:45,229 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_118046 in the backing map. Acquiring read lock for offset 353280 2024-04-12T03:12:45,229 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_100709 in the backing map. Acquiring read lock for offset 417792 2024-04-12T03:12:45,230 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_51541 in the backing map. Acquiring read lock for offset 470016 2024-04-12T03:12:45,230 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_10696 in the backing map. Acquiring read lock for offset 506880 2024-04-12T03:12:45,230 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_105059 in the backing map. Acquiring read lock for offset 411648 2024-04-12T03:12:45,230 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_55894 in the backing map. Acquiring read lock for offset 466944 2024-04-12T03:12:45,230 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_60134 in the backing map. Acquiring read lock for offset 436224 2024-04-12T03:12:45,230 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_15046 in the backing map. Acquiring read lock for offset 491520 2024-04-12T03:12:45,230 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_38723 in the backing map. Acquiring read lock for offset 442368 2024-04-12T03:12:45,230 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_92052 in the backing map. Acquiring read lock for offset 374784 2024-04-12T03:12:45,230 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_42984 in the backing map. Acquiring read lock for offset 494592 2024-04-12T03:12:45,230 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_2104 in the backing map. Acquiring read lock for offset 522240 2024-04-12T03:12:45,230 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_47217 in the backing map. Acquiring read lock for offset 488448 2024-04-12T03:12:45,230 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_96396 in the backing map. Acquiring read lock for offset 368640 2024-04-12T03:12:45,230 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_6368 in the backing map. Acquiring read lock for offset 519168 2024-04-12T03:12:45,230 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_79271 in the backing map. Acquiring read lock for offset 423936 2024-04-12T03:12:45,230 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_30210 in the backing map. Acquiring read lock for offset 497664 2024-04-12T03:12:45,231 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_83554 in the backing map. Acquiring read lock for offset 396288 2024-04-12T03:12:45,231 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_34469 in the backing map. Acquiring read lock for offset 445440 2024-04-12T03:12:45,231 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_87782 in the backing map. Acquiring read lock for offset 420864 2024-04-12T03:12:45,231 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_21414 in the backing map. Acquiring read lock for offset 503808 2024-04-12T03:12:45,231 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_66480 in the backing map. Acquiring read lock for offset 405504 2024-04-12T03:12:45,231 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_111573 in the backing map. Acquiring read lock for offset 362496 2024-04-12T03:12:45,231 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_115904 in the backing map. Acquiring read lock for offset 356352 2024-04-12T03:12:45,231 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_70796 in the backing map. Acquiring read lock for offset 430080 2024-04-12T03:12:45,231 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_75056 in the backing map. Acquiring read lock for offset 427008 2024-04-12T03:12:45,231 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_25874 in the backing map. Acquiring read lock for offset 454656 2024-04-12T03:12:45,231 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_8568 in the backing map. Acquiring read lock for offset 516096 2024-04-12T03:12:45,231 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_53682 in the backing map. Acquiring read lock for offset 482304 2024-04-12T03:12:45,232 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_102857 in the backing map. Acquiring read lock for offset 414720 2024-04-12T03:12:45,232 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_12844 in the backing map. Acquiring read lock for offset 500736 2024-04-12T03:12:45,232 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_57985 in the backing map. Acquiring read lock for offset 463872 2024-04-12T03:12:45,232 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_17132 in the backing map. Acquiring read lock for offset 460800 2024-04-12T03:12:45,232 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_107217 in the backing map. Acquiring read lock for offset 365568 2024-04-12T03:12:45,232 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2075): found block 5347477213354343993cd2ff506bfcba_62250 in the backing map. Acquiring read lock for offset 473088 2024-04-12T03:12:45,232 DEBUG [hfile-prefetch-1712891562038 {}] bucket.BucketCache(2093): File /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/699211b6-59f2-0bbd-30ca-c92e999a5a56/TestPrefetch3/5347477213354343993cd2ff506bfcba has now been fully cached. 2024-04-12T03:12:45,232 WARN [hfile-prefetch-1712891562038 {}] hfile.HFilePreadReader$1(151): Prefetch path=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/699211b6-59f2-0bbd-30ca-c92e999a5a56/TestPrefetch3/5347477213354343993cd2ff506bfcba, offset=120324, end=120324 java.lang.IllegalMonitorStateException: attempt to unlock read lock, not locked by current thread at java.util.concurrent.locks.ReentrantReadWriteLock$Sync.unmatchedUnlockException(ReentrantReadWriteLock.java:444) ~[?:1.8.0_362] at java.util.concurrent.locks.ReentrantReadWriteLock$Sync.tryReleaseShared(ReentrantReadWriteLock.java:428) ~[?:1.8.0_362] at java.util.concurrent.locks.AbstractQueuedSynchronizer.releaseShared(AbstractQueuedSynchronizer.java:1341) ~[?:1.8.0_362] at java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.unlock(ReentrantReadWriteLock.java:881) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2125) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:141) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_362] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) ~[?:1.8.0_362] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-12T03:12:45,234 DEBUG [hfile-prefetch-1712891562038 {}] hfile.PrefetchExecutor(108): Prefetch completed for 5347477213354343993cd2ff506bfcba 2024-04-12T03:12:45,298 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-04-12T03:12:45,298 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1310): Thread bucket-cache-persister started persisting bucket cache to file 2024-04-12T03:12:45,303 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1323): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-04-12T03:12:46,123 DEBUG [Time-limited test {}] bucket.BucketCache(739): removed key 5347477213354343993cd2ff506bfcba_40813 from back map with offset lock 479232 in the evict process 2024-04-12T03:12:46,123 DEBUG [Time-limited test {}] bucket.BucketCache(651): Removing file 5347477213354343993cd2ff506bfcba for region 699211b6-59f2-0bbd-30ca-c92e999a5a56 2024-04-12T03:12:46,123 INFO [Time-limited test {}] bucket.BucketCache(1534): Disabling cache 2024-04-12T03:12:46,124 INFO [Time-limited test {}] bucket.BucketCache(1558): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/699211b6-59f2-0bbd-30ca-c92e999a5a56/bucket.persistence 2024-04-12T03:12:46,124 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1094): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-04-12T03:12:46,124 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1094): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-04-12T03:12:46,124 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1094): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-04-12T03:12:46,124 INFO [Time-limited test {}] bucket.BucketCache(1564): Shutting down cache persister thread. 2024-04-12T03:12:46,124 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-04-12T03:12:46,135 DEBUG [Time-limited test {}] bucket.BucketCache(1310): Thread Time-limited test started persisting bucket cache to file 2024-04-12T03:12:46,139 DEBUG [Time-limited test {}] bucket.BucketCache(1323): Thread Time-limited test finished persisting bucket cache to file, renaming 2024-04-12T03:12:46,140 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-04-12T03:12:46,141 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-04-12T03:12:46,152 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestBucketCachePersister#testPrefetchListUponBlockEviction Thread=16 (was 15) Potentially hanging thread: hfile-prefetch-1712891564113 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=310 (was 309) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=288 (was 288), ProcessCount=171 (was 171), AvailableMemoryMB=4874 (was 4877) 2024-04-12T03:12:46,163 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestBucketCachePersister#testPrefetchPersistenceCrash Thread=16, OpenFileDescriptor=310, MaxFileDescriptor=60000, SystemLoadAverage=288, ProcessCount=171, AvailableMemoryMB=4874 2024-04-12T03:12:46,164 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-04-12T03:12:46,164 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/699211b6-59f2-0bbd-30ca-c92e999a5a56/testPrefetchPersistenceCrash 2024-04-12T03:12:46,165 INFO [Time-limited test {}] bucket.BucketCache(305): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2024-04-12T03:12:46,165 INFO [Time-limited test {}] bucket.BucketCache(393): Starting BucketCachePersisterThread 2024-04-12T03:12:46,165 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 3000 2024-04-12T03:12:46,165 INFO [Time-limited test {}] bucket.BucketCache(1344): Started retrieving bucket cache from file 2024-04-12T03:12:46,165 WARN [Time-limited test {}] bucket.BucketCache(1347): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-04-12T03:12:46,166 INFO [Time-limited test {}] bucket.BucketAllocator(339): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-04-12T03:12:46,166 INFO [Time-limited test {}] bucket.BucketCache(356): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/699211b6-59f2-0bbd-30ca-c92e999a5a56/testPrefetchPersistenceCrash, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/699211b6-59f2-0bbd-30ca-c92e999a5a56/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-04-12T03:12:46,167 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-04-12T03:12:46,170 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ca08e68c9fb045e399caffc50cb121d5 2024-04-12T03:12:46,179 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(813): Len of the biggest cell in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/699211b6-59f2-0bbd-30ca-c92e999a5a56/TestPrefetch0/ca08e68c9fb045e399caffc50cb121d5 is 243, key is aaaaaaaaaaaaaaaaaaaaaabbbababbba/mG7yno_:7/2040024791108931104/DeleteColumn/seqid=0 2024-04-12T03:12:46,184 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b133d00f90cb45d7b0bc0945c7ebd037 2024-04-12T03:12:46,192 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(813): Len of the biggest cell in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/699211b6-59f2-0bbd-30ca-c92e999a5a56/TestPrefetch1/b133d00f90cb45d7b0bc0945c7ebd037 is 248, key is aaaaaaaaaaaaaaaaaaaaaaaabaaaabbb/LK2U/-3703344447349604554/Put/seqid=0 2024-04-12T03:12:46,194 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(92): Prefetch requested for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/699211b6-59f2-0bbd-30ca-c92e999a5a56/TestPrefetch0/ca08e68c9fb045e399caffc50cb121d5, delay=972 ms 2024-04-12T03:12:47,167 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_0. 2024-04-12T03:12:47,167 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_2147. 2024-04-12T03:12:47,168 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_4335. 2024-04-12T03:12:47,168 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_6494. 2024-04-12T03:12:47,169 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_8723. 2024-04-12T03:12:47,169 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_10854. 2024-04-12T03:12:47,169 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_12944. 2024-04-12T03:12:47,169 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_15040. 2024-04-12T03:12:47,169 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_17221. 2024-04-12T03:12:47,170 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_19381. 2024-04-12T03:12:47,170 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_21478. 2024-04-12T03:12:47,170 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_23585. 2024-04-12T03:12:47,170 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_25710. 2024-04-12T03:12:47,170 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_27806. 2024-04-12T03:12:47,170 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_29911. 2024-04-12T03:12:47,170 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_32074. 2024-04-12T03:12:47,171 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_34251. 2024-04-12T03:12:47,171 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_36372. 2024-04-12T03:12:47,171 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_38465. 2024-04-12T03:12:47,171 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_40554. 2024-04-12T03:12:47,171 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_42708. 2024-04-12T03:12:47,171 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_44808. 2024-04-12T03:12:47,172 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_46895. 2024-04-12T03:12:47,172 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_49069. 2024-04-12T03:12:47,172 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_51204. 2024-04-12T03:12:47,172 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_53365. 2024-04-12T03:12:47,173 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_55501. 2024-04-12T03:12:47,173 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_57731. 2024-04-12T03:12:47,173 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_59821. 2024-04-12T03:12:47,174 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_61979. 2024-04-12T03:12:47,174 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_64102. 2024-04-12T03:12:47,174 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_66290. 2024-04-12T03:12:47,174 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_68394. 2024-04-12T03:12:47,174 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_70482. 2024-04-12T03:12:47,175 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_72609. 2024-04-12T03:12:47,175 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_74758. 2024-04-12T03:12:47,175 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_77010. 2024-04-12T03:12:47,175 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_79153. 2024-04-12T03:12:47,175 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_81301. 2024-04-12T03:12:47,176 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_83394. 2024-04-12T03:12:47,176 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_85490. 2024-04-12T03:12:47,176 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_87663. 2024-04-12T03:12:47,176 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_89782. 2024-04-12T03:12:47,176 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_91907. 2024-04-12T03:12:47,176 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_94070. 2024-04-12T03:12:47,176 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_96224. 2024-04-12T03:12:47,177 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_98375. 2024-04-12T03:12:47,177 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_100519. 2024-04-12T03:12:47,177 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_102693. 2024-04-12T03:12:47,177 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_104778. 2024-04-12T03:12:47,177 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_106908. 2024-04-12T03:12:47,178 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_109014. 2024-04-12T03:12:47,178 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_111138. 2024-04-12T03:12:47,178 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_113357. 2024-04-12T03:12:47,178 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_115478. 2024-04-12T03:12:47,178 DEBUG [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key ca08e68c9fb045e399caffc50cb121d5_117108. 2024-04-12T03:12:47,179 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2064): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/699211b6-59f2-0bbd-30ca-c92e999a5a56/TestPrefetch0/ca08e68c9fb045e399caffc50cb121d5, with total blocks 56, and data blocks 55 2024-04-12T03:12:47,179 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2068): iterating over 18 entries in the backing map 2024-04-12T03:12:47,179 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_12944 in the backing map. Acquiring read lock for offset 500736 2024-04-12T03:12:47,179 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_4335 in the backing map. Acquiring read lock for offset 516096 2024-04-12T03:12:47,179 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_8723 in the backing map. Acquiring read lock for offset 506880 2024-04-12T03:12:47,179 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_40554 in the backing map. Acquiring read lock for offset 485376 2024-04-12T03:12:47,179 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_44808 in the backing map. Acquiring read lock for offset 473088 2024-04-12T03:12:47,179 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_0 in the backing map. Acquiring read lock for offset 525312 2024-04-12T03:12:47,179 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_27806 in the backing map. Acquiring read lock for offset 488448 2024-04-12T03:12:47,179 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_36372 in the backing map. Acquiring read lock for offset 491520 2024-04-12T03:12:47,179 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_10854 in the backing map. Acquiring read lock for offset 509952 2024-04-12T03:12:47,179 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_15040 in the backing map. Acquiring read lock for offset 519168 2024-04-12T03:12:47,179 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_23585 in the backing map. Acquiring read lock for offset 494592 2024-04-12T03:12:47,179 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_2147 in the backing map. Acquiring read lock for offset 522240 2024-04-12T03:12:47,179 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_6494 in the backing map. Acquiring read lock for offset 513024 2024-04-12T03:12:47,179 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_42708 in the backing map. Acquiring read lock for offset 479232 2024-04-12T03:12:47,179 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_46895 in the backing map. Acquiring read lock for offset 476160 2024-04-12T03:12:47,179 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_25710 in the backing map. Acquiring read lock for offset 503808 2024-04-12T03:12:47,180 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_34251 in the backing map. Acquiring read lock for offset 497664 2024-04-12T03:12:47,180 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_38465 in the backing map. Acquiring read lock for offset 482304 2024-04-12T03:12:47,180 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2096): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/699211b6-59f2-0bbd-30ca-c92e999a5a56/TestPrefetch0/ca08e68c9fb045e399caffc50cb121d5, but only 18 data blocks were cached. Total data blocks for file: 55. Checking for blocks pending cache in cache writer queue. 2024-04-12T03:12:47,180 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2105): There are still blocks pending caching for file ca08e68c9fb045e399caffc50cb121d5. Will sleep 100ms and try the verification again. 2024-04-12T03:12:47,280 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2064): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/699211b6-59f2-0bbd-30ca-c92e999a5a56/TestPrefetch0/ca08e68c9fb045e399caffc50cb121d5, with total blocks 56, and data blocks 55 2024-04-12T03:12:47,280 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2068): iterating over 56 entries in the backing map 2024-04-12T03:12:47,280 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_12944 in the backing map. Acquiring read lock for offset 500736 2024-04-12T03:12:47,280 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_66290 in the backing map. Acquiring read lock for offset 414720 2024-04-12T03:12:47,280 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_17221 in the backing map. Acquiring read lock for offset 470016 2024-04-12T03:12:47,281 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_115478 in the backing map. Acquiring read lock for offset 359424 2024-04-12T03:12:47,281 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_70482 in the backing map. Acquiring read lock for offset 417792 2024-04-12T03:12:47,281 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_74758 in the backing map. Acquiring read lock for offset 439296 2024-04-12T03:12:47,281 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_21478 in the backing map. Acquiring read lock for offset 463872 2024-04-12T03:12:47,281 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_53365 in the backing map. Acquiring read lock for offset 451584 2024-04-12T03:12:47,281 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_4335 in the backing map. Acquiring read lock for offset 516096 2024-04-12T03:12:47,281 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_102693 in the backing map. Acquiring read lock for offset 384000 2024-04-12T03:12:47,281 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_57731 in the backing map. Acquiring read lock for offset 427008 2024-04-12T03:12:47,281 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_106908 in the backing map. Acquiring read lock for offset 374784 2024-04-12T03:12:47,281 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_8723 in the backing map. Acquiring read lock for offset 506880 2024-04-12T03:12:47,281 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_61979 in the backing map. Acquiring read lock for offset 420864 2024-04-12T03:12:47,281 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_111138 in the backing map. Acquiring read lock for offset 368640 2024-04-12T03:12:47,281 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_40554 in the backing map. Acquiring read lock for offset 485376 2024-04-12T03:12:47,281 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_89782 in the backing map. Acquiring read lock for offset 365568 2024-04-12T03:12:47,281 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_44808 in the backing map. Acquiring read lock for offset 473088 2024-04-12T03:12:47,281 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_94070 in the backing map. Acquiring read lock for offset 399360 2024-04-12T03:12:47,281 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_49069 in the backing map. Acquiring read lock for offset 454656 2024-04-12T03:12:47,282 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_0 in the backing map. Acquiring read lock for offset 525312 2024-04-12T03:12:47,282 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_98375 in the backing map. Acquiring read lock for offset 387072 2024-04-12T03:12:47,282 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_27806 in the backing map. Acquiring read lock for offset 488448 2024-04-12T03:12:47,282 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_77010 in the backing map. Acquiring read lock for offset 402432 2024-04-12T03:12:47,282 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_32074 in the backing map. Acquiring read lock for offset 457728 2024-04-12T03:12:47,282 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_81301 in the backing map. Acquiring read lock for offset 433152 2024-04-12T03:12:47,282 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_85490 in the backing map. Acquiring read lock for offset 408576 2024-04-12T03:12:47,282 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_36372 in the backing map. Acquiring read lock for offset 491520 2024-04-12T03:12:47,282 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_10854 in the backing map. Acquiring read lock for offset 509952 2024-04-12T03:12:47,282 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_64102 in the backing map. Acquiring read lock for offset 442368 2024-04-12T03:12:47,282 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_15040 in the backing map. Acquiring read lock for offset 519168 2024-04-12T03:12:47,282 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_113357 in the backing map. Acquiring read lock for offset 380928 2024-04-12T03:12:47,282 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_68394 in the backing map. Acquiring read lock for offset 423936 2024-04-12T03:12:47,282 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_19381 in the backing map. Acquiring read lock for offset 466944 2024-04-12T03:12:47,282 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_72609 in the backing map. Acquiring read lock for offset 411648 2024-04-12T03:12:47,282 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_23585 in the backing map. Acquiring read lock for offset 494592 2024-04-12T03:12:47,282 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_2147 in the backing map. Acquiring read lock for offset 522240 2024-04-12T03:12:47,282 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_55501 in the backing map. Acquiring read lock for offset 448512 2024-04-12T03:12:47,282 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_100519 in the backing map. Acquiring read lock for offset 362496 2024-04-12T03:12:47,282 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_104778 in the backing map. Acquiring read lock for offset 377856 2024-04-12T03:12:47,283 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_6494 in the backing map. Acquiring read lock for offset 513024 2024-04-12T03:12:47,283 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_59821 in the backing map. Acquiring read lock for offset 445440 2024-04-12T03:12:47,283 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_109014 in the backing map. Acquiring read lock for offset 371712 2024-04-12T03:12:47,283 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_87663 in the backing map. Acquiring read lock for offset 390144 2024-04-12T03:12:47,283 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_91907 in the backing map. Acquiring read lock for offset 405504 2024-04-12T03:12:47,283 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_42708 in the backing map. Acquiring read lock for offset 479232 2024-04-12T03:12:47,283 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_46895 in the backing map. Acquiring read lock for offset 476160 2024-04-12T03:12:47,283 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_51204 in the backing map. Acquiring read lock for offset 430080 2024-04-12T03:12:47,283 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_96224 in the backing map. Acquiring read lock for offset 393216 2024-04-12T03:12:47,283 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_25710 in the backing map. Acquiring read lock for offset 503808 2024-04-12T03:12:47,283 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_29911 in the backing map. Acquiring read lock for offset 460800 2024-04-12T03:12:47,283 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_79153 in the backing map. Acquiring read lock for offset 436224 2024-04-12T03:12:47,283 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_83394 in the backing map. Acquiring read lock for offset 396288 2024-04-12T03:12:47,283 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_34251 in the backing map. Acquiring read lock for offset 497664 2024-04-12T03:12:47,283 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2075): found block ca08e68c9fb045e399caffc50cb121d5_38465 in the backing map. Acquiring read lock for offset 482304 2024-04-12T03:12:47,283 DEBUG [hfile-prefetch-1712891560758 {}] bucket.BucketCache(2093): File /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/699211b6-59f2-0bbd-30ca-c92e999a5a56/TestPrefetch0/ca08e68c9fb045e399caffc50cb121d5 has now been fully cached. 2024-04-12T03:12:47,284 WARN [hfile-prefetch-1712891560758 {}] hfile.HFilePreadReader$1(151): Prefetch path=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/699211b6-59f2-0bbd-30ca-c92e999a5a56/TestPrefetch0/ca08e68c9fb045e399caffc50cb121d5, offset=117401, end=117401 java.lang.IllegalMonitorStateException: attempt to unlock read lock, not locked by current thread at java.util.concurrent.locks.ReentrantReadWriteLock$Sync.unmatchedUnlockException(ReentrantReadWriteLock.java:444) ~[?:1.8.0_362] at java.util.concurrent.locks.ReentrantReadWriteLock$Sync.tryReleaseShared(ReentrantReadWriteLock.java:428) ~[?:1.8.0_362] at java.util.concurrent.locks.AbstractQueuedSynchronizer.releaseShared(AbstractQueuedSynchronizer.java:1341) ~[?:1.8.0_362] at java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.unlock(ReentrantReadWriteLock.java:881) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2125) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:141) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_362] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) ~[?:1.8.0_362] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-12T03:12:47,284 DEBUG [hfile-prefetch-1712891560758 {}] hfile.PrefetchExecutor(108): Prefetch completed for ca08e68c9fb045e399caffc50cb121d5 2024-04-12T03:12:48,196 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(92): Prefetch requested for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/699211b6-59f2-0bbd-30ca-c92e999a5a56/TestPrefetch1/b133d00f90cb45d7b0bc0945c7ebd037, delay=925 ms 2024-04-12T03:12:49,122 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_0. 2024-04-12T03:12:49,123 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_2231. 2024-04-12T03:12:49,123 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_4361. 2024-04-12T03:12:49,123 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_6520. 2024-04-12T03:12:49,124 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_8716. 2024-04-12T03:12:49,124 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_10813. 2024-04-12T03:12:49,124 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_13024. 2024-04-12T03:12:49,124 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_15173. 2024-04-12T03:12:49,124 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_17280. 2024-04-12T03:12:49,124 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_19416. 2024-04-12T03:12:49,125 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_21634. 2024-04-12T03:12:49,125 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_23795. 2024-04-12T03:12:49,125 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_25904. 2024-04-12T03:12:49,125 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_28035. 2024-04-12T03:12:49,126 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_30224. 2024-04-12T03:12:49,126 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_32365. 2024-04-12T03:12:49,126 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_34524. 2024-04-12T03:12:49,126 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_36671. 2024-04-12T03:12:49,126 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_38830. 2024-04-12T03:12:49,127 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_41044. 2024-04-12T03:12:49,127 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_43222. 2024-04-12T03:12:49,127 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_45377. 2024-04-12T03:12:49,127 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_47534. 2024-04-12T03:12:49,127 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_49770. 2024-04-12T03:12:49,127 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_51858. 2024-04-12T03:12:49,127 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_53944. 2024-04-12T03:12:49,128 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_56101. 2024-04-12T03:12:49,128 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_58203. 2024-04-12T03:12:49,128 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_60312. 2024-04-12T03:12:49,128 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_62497. 2024-04-12T03:12:49,128 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_64671. 2024-04-12T03:12:49,128 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_66822. 2024-04-12T03:12:49,129 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_68931. 2024-04-12T03:12:49,129 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_71072. 2024-04-12T03:12:49,129 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_73174. 2024-04-12T03:12:49,129 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_75421. 2024-04-12T03:12:49,130 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_77619. 2024-04-12T03:12:49,130 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_79825. 2024-04-12T03:12:49,130 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_82017. 2024-04-12T03:12:49,130 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_84184. 2024-04-12T03:12:49,131 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_86365. 2024-04-12T03:12:49,131 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_88453. 2024-04-12T03:12:49,131 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_90668. 2024-04-12T03:12:49,131 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_92846. 2024-04-12T03:12:49,131 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_95022. 2024-04-12T03:12:49,131 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_97179. 2024-04-12T03:12:49,131 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_99365. 2024-04-12T03:12:49,132 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_101528. 2024-04-12T03:12:49,132 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_103730. 2024-04-12T03:12:49,132 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_105911. 2024-04-12T03:12:49,132 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_108004. 2024-04-12T03:12:49,132 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_110170. 2024-04-12T03:12:49,132 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_112290. 2024-04-12T03:12:49,132 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_114409. 2024-04-12T03:12:49,133 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_116540. 2024-04-12T03:12:49,133 DEBUG [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(100): No entry in the backing map for cache key b133d00f90cb45d7b0bc0945c7ebd037_117912. 2024-04-12T03:12:49,133 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2064): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/699211b6-59f2-0bbd-30ca-c92e999a5a56/TestPrefetch1/b133d00f90cb45d7b0bc0945c7ebd037, with total blocks 56, and data blocks 55 2024-04-12T03:12:49,133 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2068): iterating over 69 entries in the backing map 2024-04-12T03:12:49,133 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_34524 in the backing map. Acquiring read lock for offset 316416 2024-04-12T03:12:49,133 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_30224 in the backing map. Acquiring read lock for offset 322560 2024-04-12T03:12:49,133 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_25904 in the backing map. Acquiring read lock for offset 328704 2024-04-12T03:12:49,133 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_19416 in the backing map. Acquiring read lock for offset 331776 2024-04-12T03:12:49,133 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_15173 in the backing map. Acquiring read lock for offset 337920 2024-04-12T03:12:49,133 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_6520 in the backing map. Acquiring read lock for offset 347136 2024-04-12T03:12:49,133 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_2231 in the backing map. Acquiring read lock for offset 350208 2024-04-12T03:12:49,133 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_0 in the backing map. Acquiring read lock for offset 353280 2024-04-12T03:12:49,133 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_36671 in the backing map. Acquiring read lock for offset 319488 2024-04-12T03:12:49,133 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_28035 in the backing map. Acquiring read lock for offset 325632 2024-04-12T03:12:49,133 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_23795 in the backing map. Acquiring read lock for offset 334848 2024-04-12T03:12:49,133 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_17280 in the backing map. Acquiring read lock for offset 340992 2024-04-12T03:12:49,134 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_4361 in the backing map. Acquiring read lock for offset 344064 2024-04-12T03:12:49,134 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2096): Prefetch executor completed for /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/699211b6-59f2-0bbd-30ca-c92e999a5a56/TestPrefetch1/b133d00f90cb45d7b0bc0945c7ebd037, but only 13 data blocks were cached. Total data blocks for file: 55. Checking for blocks pending cache in cache writer queue. 2024-04-12T03:12:49,134 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2105): There are still blocks pending caching for file b133d00f90cb45d7b0bc0945c7ebd037. Will sleep 100ms and try the verification again. 2024-04-12T03:12:49,165 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-04-12T03:12:49,166 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1310): Thread bucket-cache-persister started persisting bucket cache to file 2024-04-12T03:12:49,171 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1323): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-04-12T03:12:49,234 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2064): Notifying caching completed for file /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/699211b6-59f2-0bbd-30ca-c92e999a5a56/TestPrefetch1/b133d00f90cb45d7b0bc0945c7ebd037, with total blocks 56, and data blocks 55 2024-04-12T03:12:49,234 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2068): iterating over 112 entries in the backing map 2024-04-12T03:12:49,234 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_38830 in the backing map. Acquiring read lock for offset 264192 2024-04-12T03:12:49,234 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_79825 in the backing map. Acquiring read lock for offset 239616 2024-04-12T03:12:49,234 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_116540 in the backing map. Acquiring read lock for offset 199680 2024-04-12T03:12:49,234 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_75421 in the backing map. Acquiring read lock for offset 242688 2024-04-12T03:12:49,234 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_112290 in the backing map. Acquiring read lock for offset 215040 2024-04-12T03:12:49,234 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_34524 in the backing map. Acquiring read lock for offset 316416 2024-04-12T03:12:49,234 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_71072 in the backing map. Acquiring read lock for offset 294912 2024-04-12T03:12:49,235 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_108004 in the backing map. Acquiring read lock for offset 190464 2024-04-12T03:12:49,235 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_30224 in the backing map. Acquiring read lock for offset 322560 2024-04-12T03:12:49,235 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_25904 in the backing map. Acquiring read lock for offset 328704 2024-04-12T03:12:49,235 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_103730 in the backing map. Acquiring read lock for offset 208896 2024-04-12T03:12:49,235 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_66822 in the backing map. Acquiring read lock for offset 288768 2024-04-12T03:12:49,235 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_99365 in the backing map. Acquiring read lock for offset 224256 2024-04-12T03:12:49,235 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_62497 in the backing map. Acquiring read lock for offset 248832 2024-04-12T03:12:49,235 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_21634 in the backing map. Acquiring read lock for offset 270336 2024-04-12T03:12:49,235 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_60312 in the backing map. Acquiring read lock for offset 297984 2024-04-12T03:12:49,235 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_97179 in the backing map. Acquiring read lock for offset 221184 2024-04-12T03:12:49,235 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_19416 in the backing map. Acquiring read lock for offset 331776 2024-04-12T03:12:49,235 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_56101 in the backing map. Acquiring read lock for offset 251904 2024-04-12T03:12:49,235 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_15173 in the backing map. Acquiring read lock for offset 337920 2024-04-12T03:12:49,235 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_92846 in the backing map. Acquiring read lock for offset 230400 2024-04-12T03:12:49,235 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_10813 in the backing map. Acquiring read lock for offset 276480 2024-04-12T03:12:49,235 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_51858 in the backing map. Acquiring read lock for offset 307200 2024-04-12T03:12:49,235 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_47534 in the backing map. Acquiring read lock for offset 301056 2024-04-12T03:12:49,235 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_88453 in the backing map. Acquiring read lock for offset 193536 2024-04-12T03:12:49,235 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_6520 in the backing map. Acquiring read lock for offset 347136 2024-04-12T03:12:49,235 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_2231 in the backing map. Acquiring read lock for offset 350208 2024-04-12T03:12:49,235 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_84184 in the backing map. Acquiring read lock for offset 205824 2024-04-12T03:12:49,235 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_43222 in the backing map. Acquiring read lock for offset 258048 2024-04-12T03:12:49,236 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_0 in the backing map. Acquiring read lock for offset 353280 2024-04-12T03:12:49,236 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_77619 in the backing map. Acquiring read lock for offset 282624 2024-04-12T03:12:49,236 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_36671 in the backing map. Acquiring read lock for offset 319488 2024-04-12T03:12:49,236 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_114409 in the backing map. Acquiring read lock for offset 184320 2024-04-12T03:12:49,236 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_32365 in the backing map. Acquiring read lock for offset 267264 2024-04-12T03:12:49,236 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_110170 in the backing map. Acquiring read lock for offset 187392 2024-04-12T03:12:49,236 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_105911 in the backing map. Acquiring read lock for offset 202752 2024-04-12T03:12:49,236 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_73174 in the backing map. Acquiring read lock for offset 245760 2024-04-12T03:12:49,236 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_68931 in the backing map. Acquiring read lock for offset 285696 2024-04-12T03:12:49,236 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_28035 in the backing map. Acquiring read lock for offset 325632 2024-04-12T03:12:49,236 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_64671 in the backing map. Acquiring read lock for offset 291840 2024-04-12T03:12:49,236 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_101528 in the backing map. Acquiring read lock for offset 218112 2024-04-12T03:12:49,236 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_23795 in the backing map. Acquiring read lock for offset 334848 2024-04-12T03:12:49,236 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_95022 in the backing map. Acquiring read lock for offset 227328 2024-04-12T03:12:49,236 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_58203 in the backing map. Acquiring read lock for offset 304128 2024-04-12T03:12:49,237 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_17280 in the backing map. Acquiring read lock for offset 340992 2024-04-12T03:12:49,237 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_53944 in the backing map. Acquiring read lock for offset 254976 2024-04-12T03:12:49,237 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_13024 in the backing map. Acquiring read lock for offset 273408 2024-04-12T03:12:49,237 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_90668 in the backing map. Acquiring read lock for offset 236544 2024-04-12T03:12:49,237 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_49770 in the backing map. Acquiring read lock for offset 313344 2024-04-12T03:12:49,237 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_8716 in the backing map. Acquiring read lock for offset 279552 2024-04-12T03:12:49,237 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_45377 in the backing map. Acquiring read lock for offset 310272 2024-04-12T03:12:49,237 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_86365 in the backing map. Acquiring read lock for offset 196608 2024-04-12T03:12:49,237 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_4361 in the backing map. Acquiring read lock for offset 344064 2024-04-12T03:12:49,237 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_41044 in the backing map. Acquiring read lock for offset 261120 2024-04-12T03:12:49,237 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2075): found block b133d00f90cb45d7b0bc0945c7ebd037_82017 in the backing map. Acquiring read lock for offset 233472 2024-04-12T03:12:49,237 DEBUG [hfile-prefetch-1712891564113 {}] bucket.BucketCache(2093): File /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/699211b6-59f2-0bbd-30ca-c92e999a5a56/TestPrefetch1/b133d00f90cb45d7b0bc0945c7ebd037 has now been fully cached. 2024-04-12T03:12:49,238 WARN [hfile-prefetch-1712891564113 {}] hfile.HFilePreadReader$1(151): Prefetch path=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/699211b6-59f2-0bbd-30ca-c92e999a5a56/TestPrefetch1/b133d00f90cb45d7b0bc0945c7ebd037, offset=118205, end=118205 java.lang.IllegalMonitorStateException: attempt to unlock read lock, not locked by current thread at java.util.concurrent.locks.ReentrantReadWriteLock$Sync.unmatchedUnlockException(ReentrantReadWriteLock.java:444) ~[?:1.8.0_362] at java.util.concurrent.locks.ReentrantReadWriteLock$Sync.tryReleaseShared(ReentrantReadWriteLock.java:428) ~[?:1.8.0_362] at java.util.concurrent.locks.AbstractQueuedSynchronizer.releaseShared(AbstractQueuedSynchronizer.java:1341) ~[?:1.8.0_362] at java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.unlock(ReentrantReadWriteLock.java:881) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2125) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:141) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_362] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) ~[?:1.8.0_362] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_362] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2024-04-12T03:12:49,238 DEBUG [hfile-prefetch-1712891564113 {}] hfile.PrefetchExecutor(108): Prefetch completed for b133d00f90cb45d7b0bc0945c7ebd037 2024-04-12T03:12:53,198 INFO [Time-limited test {}] bucket.BucketCache(1534): Disabling cache 2024-04-12T03:12:53,199 INFO [Time-limited test {}] bucket.BucketCache(1558): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/699211b6-59f2-0bbd-30ca-c92e999a5a56/bucket.persistence 2024-04-12T03:12:53,199 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1094): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-04-12T03:12:53,199 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1094): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-04-12T03:12:53,199 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1094): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-04-12T03:12:53,199 INFO [Time-limited test {}] bucket.BucketCache(1564): Shutting down cache persister thread. 2024-04-12T03:12:53,200 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-04-12T03:12:53,210 DEBUG [Time-limited test {}] bucket.BucketCache(1310): Thread Time-limited test started persisting bucket cache to file 2024-04-12T03:12:53,215 DEBUG [Time-limited test {}] bucket.BucketCache(1323): Thread Time-limited test finished persisting bucket cache to file, renaming 2024-04-12T03:12:53,216 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-04-12T03:12:53,217 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-04-12T03:12:53,227 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestBucketCachePersister#testPrefetchPersistenceCrash Thread=17 (was 16) Potentially hanging thread: hfile-prefetch-1712891566194 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=312 (was 310) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=275 (was 288), ProcessCount=171 (was 171), AvailableMemoryMB=4870 (was 4874)