2023-10-04 08:06:57,906 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@233795b6 2023-10-04 08:06:57,919 main DEBUG Took 0.011233 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2023-10-04 08:06:57,920 main DEBUG PluginManager 'Core' found 129 plugins 2023-10-04 08:06:57,920 main DEBUG PluginManager 'Level' found 0 plugins 2023-10-04 08:06:57,921 main DEBUG PluginManager 'Lookup' found 16 plugins 2023-10-04 08:06:57,923 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-04 08:06:57,933 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2023-10-04 08:06:57,948 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-04 08:06:57,950 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-04 08:06:57,950 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-04 08:06:57,950 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-04 08:06:57,951 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-04 08:06:57,951 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-04 08:06:57,952 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-04 08:06:57,953 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-04 08:06:57,953 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-04 08:06:57,954 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-04 08:06:57,954 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-04 08:06:57,955 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-04 08:06:57,955 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-04 08:06:57,955 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-04 08:06:57,956 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-04 08:06:57,956 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-04 08:06:57,956 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-04 08:06:57,957 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-04 08:06:57,957 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-04 08:06:57,957 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-04 08:06:57,958 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-04 08:06:57,958 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-04 08:06:57,958 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-04 08:06:57,959 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-04 08:06:57,959 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-04 08:06:57,959 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2023-10-04 08:06:57,961 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-04 08:06:57,963 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2023-10-04 08:06:57,964 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2023-10-04 08:06:57,965 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2023-10-04 08:06:57,966 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2023-10-04 08:06:57,966 main DEBUG PluginManager 'Converter' found 47 plugins 2023-10-04 08:06:57,977 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2023-10-04 08:06:57,980 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2023-10-04 08:06:57,981 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2023-10-04 08:06:57,981 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2023-10-04 08:06:57,985 main DEBUG createAppenders(={Console}) 2023-10-04 08:06:57,985 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@233795b6 initialized 2023-10-04 08:06:57,986 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@233795b6 2023-10-04 08:06:57,986 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@233795b6 OK. 2023-10-04 08:06:57,986 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2023-10-04 08:06:57,987 main DEBUG OutputStream closed 2023-10-04 08:06:57,987 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2023-10-04 08:06:57,987 main DEBUG Appender DefaultConsole-1 stopped with status true 2023-10-04 08:06:57,987 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@f5ac9e4 OK 2023-10-04 08:06:58,030 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586 2023-10-04 08:06:58,032 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=StatusLogger 2023-10-04 08:06:58,033 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=ContextSelector 2023-10-04 08:06:58,034 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name= 2023-10-04 08:06:58,034 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.directory 2023-10-04 08:06:58,035 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2023-10-04 08:06:58,035 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.zookeeper 2023-10-04 08:06:58,035 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2023-10-04 08:06:58,036 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2023-10-04 08:06:58,036 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2023-10-04 08:06:58,036 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase 2023-10-04 08:06:58,036 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop 2023-10-04 08:06:58,037 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2023-10-04 08:06:58,037 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2023-10-04 08:06:58,037 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2023-10-04 08:06:58,038 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2023-10-04 08:06:58,038 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2023-10-04 08:06:58,039 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Appenders,name=Console 2023-10-04 08:06:58,041 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-10-04 08:06:58,042 main DEBUG Reconfiguration complete for context[name=1b6d3586] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.6.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@32b260fa) with optional ClassLoader: null 2023-10-04 08:06:58,042 main DEBUG Shutdown hook enabled. Registering a new one. 2023-10-04 08:06:58,043 main DEBUG LoggerContext[name=1b6d3586, org.apache.logging.log4j.core.LoggerContext@32b260fa] started OK. 2023-10-04T08:06:58,060 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.bucket.TestVerifyBucketCacheFile timeout: 26 mins 2023-10-04 08:06:58,064 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2023-10-04 08:06:58,064 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-10-04T08:06:58,103 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestVerifyBucketCacheFile#testBucketCacheRecovery[0: blockSize=8,192, bucketSizes=null] Thread=9, OpenFileDescriptor=271, MaxFileDescriptor=60000, SystemLoadAverage=472, ProcessCount=167, AvailableMemoryMB=6727 2023-10-04T08:06:58,380 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b75dca16-01bf-e208-480f-e18a7611895e 2023-10-04T08:06:58,468 WARN [Time-limited test {}] util.NativeCodeLoader(62): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2023-10-04T08:06:58,838 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-10-04T08:06:58,865 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b75dca16-01bf-e208-480f-e18a7611895e/bucket.cache 2023-10-04T08:06:58,867 INFO [Time-limited test {}] bucket.BucketCache(293): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2023-10-04T08:06:58,878 INFO [Time-limited test {}] bucket.BucketAllocator(339): Cache totalSize=31518720, buckets=15, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2023-10-04T08:06:58,879 INFO [Time-limited test {}] bucket.BucketCachePersister(35): BucketCachePersister started with interval: 9223372036854775807 2023-10-04T08:06:58,881 INFO [Time-limited test {}] bucket.BucketCache(340): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b75dca16-01bf-e208-480f-e18a7611895e/bucket.cache, capacity=32 MB, blockSize=8 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b75dca16-01bf-e208-480f-e18a7611895e/bucket.persistence1696406818850, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2023-10-04T08:06:58,901 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2023-10-04T08:06:59,506 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b75dca16-01bf-e208-480f-e18a7611895e/bucket.cache 2023-10-04T08:06:59,506 INFO [Time-limited test {}] bucket.BucketCache(293): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2023-10-04T08:06:59,506 INFO [Time-limited test {}] bucket.BucketAllocator(339): Cache totalSize=31518720, buckets=15, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2023-10-04T08:06:59,506 INFO [Time-limited test {}] bucket.BucketCachePersister(35): BucketCachePersister started with interval: 9223372036854775807 2023-10-04T08:06:59,518 WARN [Time-limited test {}] bucket.BucketCache(1372): Checksum for cache file failed. We need to validate each cache key in the backing map. This may take some time... 2023-10-04T08:06:59,519 DEBUG [Time-limited test {}] bucket.BucketCache(1380): Check for key 1047629588360518427_0 failed. Removing it from map. 2023-10-04T08:06:59,519 INFO [Time-limited test {}] bucket.BucketCache(1385): Finished validating 3 keys in the backing map. Recovered: 2. This took 1ms. 2023-10-04T08:06:59,519 INFO [Time-limited test {}] bucket.BucketAllocator(339): Cache totalSize=31518720, buckets=15, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2023-10-04T08:06:59,520 INFO [Time-limited test {}] bucket.BucketCache(340): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b75dca16-01bf-e208-480f-e18a7611895e/bucket.cache, capacity=32 MB, blockSize=8 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b75dca16-01bf-e208-480f-e18a7611895e/bucket.persistence1696406818850, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2023-10-04T08:06:59,543 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestVerifyBucketCacheFile#testBucketCacheRecovery[0: blockSize=8,192, bucketSizes=null] Thread=21 (was 9) Potentially hanging thread: BucketCacheStatsExecutor sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Time-limited test-BucketCacheWriter-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403) org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.getRAMQueueEntries(BucketCache.java:1247) org.apache.hadoop.hbase.io.hfile.bucket.BucketCache$WriterThread.run(BucketCache.java:1026) Potentially hanging thread: BucketCacheStatsExecutor sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Time-limited test-BucketCacheWriter-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403) org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.getRAMQueueEntries(BucketCache.java:1247) org.apache.hadoop.hbase.io.hfile.bucket.BucketCache$WriterThread.run(BucketCache.java:1026) Potentially hanging thread: bucket-cache-persister java.lang.Thread.sleep(Native Method) org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:41) Potentially hanging thread: Time-limited test-BucketCacheWriter-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403) org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.getRAMQueueEntries(BucketCache.java:1247) org.apache.hadoop.hbase.io.hfile.bucket.BucketCache$WriterThread.run(BucketCache.java:1026) Potentially hanging thread: bucket-cache-persister java.lang.Thread.sleep(Native Method) org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:41) Potentially hanging thread: Time-limited test-BucketCacheWriter-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403) org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.getRAMQueueEntries(BucketCache.java:1247) org.apache.hadoop.hbase.io.hfile.bucket.BucketCache$WriterThread.run(BucketCache.java:1026) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.lang.Object.wait(Native Method) java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:144) java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:165) org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:3712) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Time-limited test-BucketCacheWriter-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403) org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.getRAMQueueEntries(BucketCache.java:1247) org.apache.hadoop.hbase.io.hfile.bucket.BucketCache$WriterThread.run(BucketCache.java:1026) Potentially hanging thread: Time-limited test-BucketCacheWriter-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403) org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.getRAMQueueEntries(BucketCache.java:1247) org.apache.hadoop.hbase.io.hfile.bucket.BucketCache$WriterThread.run(BucketCache.java:1026) Potentially hanging thread: process reaper sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=280 (was 271) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=472 (was 472), ProcessCount=167 (was 167), AvailableMemoryMB=6637 (was 6727) 2023-10-04T08:06:59,550 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestVerifyBucketCacheFile#testRetrieveFromFile[0: blockSize=8,192, bucketSizes=null] Thread=21, OpenFileDescriptor=280, MaxFileDescriptor=60000, SystemLoadAverage=472, ProcessCount=167, AvailableMemoryMB=6636 2023-10-04T08:06:59,562 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b75dca16-01bf-e208-480f-e18a7611895e/hadoop-log-dir so I do NOT create it in target/test-data/d2d506c3-d2e6-860c-0577-0222c0ee6d4f 2023-10-04T08:06:59,562 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.log.dir property value differs in configuration and system: Configuration=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/../logs while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b75dca16-01bf-e208-480f-e18a7611895e/hadoop-log-dir Erasing configuration value by system value. 2023-10-04T08:06:59,562 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b75dca16-01bf-e208-480f-e18a7611895e/hadoop-tmp-dir so I do NOT create it in target/test-data/d2d506c3-d2e6-860c-0577-0222c0ee6d4f 2023-10-04T08:06:59,562 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.tmp.dir property value differs in configuration and system: Configuration=/tmp/hadoop-jenkins while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b75dca16-01bf-e208-480f-e18a7611895e/hadoop-tmp-dir Erasing configuration value by system value. 2023-10-04T08:06:59,563 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2d506c3-d2e6-860c-0577-0222c0ee6d4f 2023-10-04T08:06:59,563 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-10-04T08:06:59,574 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2d506c3-d2e6-860c-0577-0222c0ee6d4f/bucket.cache 2023-10-04T08:06:59,574 INFO [Time-limited test {}] bucket.BucketCache(293): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2023-10-04T08:06:59,574 INFO [Time-limited test {}] bucket.BucketAllocator(339): Cache totalSize=31518720, buckets=15, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2023-10-04T08:06:59,574 INFO [Time-limited test {}] bucket.BucketCachePersister(35): BucketCachePersister started with interval: 1000 2023-10-04T08:06:59,575 INFO [Time-limited test {}] bucket.BucketCache(340): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2d506c3-d2e6-860c-0577-0222c0ee6d4f/bucket.cache, capacity=32 MB, blockSize=8 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2d506c3-d2e6-860c-0577-0222c0ee6d4f/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2023-10-04T08:06:59,676 INFO [Time-limited test {}] bucket.BucketCache(1441): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2d506c3-d2e6-860c-0577-0222c0ee6d4f/bucket.persistence 2023-10-04T08:06:59,676 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1040): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2023-10-04T08:06:59,676 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1040): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2023-10-04T08:06:59,676 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1040): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2023-10-04T08:06:59,677 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(50): InterruptedException in BucketCachePersister sleep interrupted 2023-10-04T08:06:59,690 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2d506c3-d2e6-860c-0577-0222c0ee6d4f/bucket.cache 2023-10-04T08:06:59,691 INFO [Time-limited test {}] bucket.BucketCache(293): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2023-10-04T08:06:59,691 INFO [Time-limited test {}] bucket.BucketAllocator(339): Cache totalSize=31518720, buckets=15, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2023-10-04T08:06:59,691 INFO [Time-limited test {}] bucket.BucketCachePersister(35): BucketCachePersister started with interval: 1000 2023-10-04T08:06:59,693 INFO [Time-limited test {}] bucket.BucketAllocator(339): Cache totalSize=31518720, buckets=15, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2023-10-04T08:06:59,694 INFO [Time-limited test {}] bucket.BucketCache(340): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2d506c3-d2e6-860c-0577-0222c0ee6d4f/bucket.cache, capacity=32 MB, blockSize=8 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2d506c3-d2e6-860c-0577-0222c0ee6d4f/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2023-10-04T08:06:59,694 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1040): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2023-10-04T08:06:59,694 INFO [Time-limited test {}] bucket.BucketCache(1441): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2d506c3-d2e6-860c-0577-0222c0ee6d4f/bucket.persistence 2023-10-04T08:06:59,694 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1040): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2023-10-04T08:06:59,694 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1040): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2023-10-04T08:06:59,695 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(50): InterruptedException in BucketCachePersister sleep interrupted 2023-10-04T08:06:59,707 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2d506c3-d2e6-860c-0577-0222c0ee6d4f/bucket.cache 2023-10-04T08:06:59,708 INFO [Time-limited test {}] bucket.BucketCache(293): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2023-10-04T08:06:59,708 INFO [Time-limited test {}] bucket.BucketAllocator(339): Cache totalSize=31518720, buckets=15, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2023-10-04T08:06:59,708 INFO [Time-limited test {}] bucket.BucketCachePersister(35): BucketCachePersister started with interval: 1000 2023-10-04T08:06:59,710 WARN [Time-limited test {}] bucket.BucketCache(1372): Checksum for cache file failed. We need to validate each cache key in the backing map. This may take some time... 2023-10-04T08:06:59,710 DEBUG [Time-limited test {}] bucket.BucketCache(1380): Check for key 7141140069765854841_0 failed. Removing it from map. 2023-10-04T08:06:59,710 INFO [Time-limited test {}] bucket.BucketCache(1385): Finished validating 1 keys in the backing map. Recovered: 0. This took 0ms. 2023-10-04T08:06:59,711 INFO [Time-limited test {}] bucket.BucketAllocator(339): Cache totalSize=31518720, buckets=15, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2023-10-04T08:06:59,711 INFO [Time-limited test {}] bucket.BucketCache(340): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2d506c3-d2e6-860c-0577-0222c0ee6d4f/bucket.cache, capacity=32 MB, blockSize=8 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2d506c3-d2e6-860c-0577-0222c0ee6d4f/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2023-10-04T08:06:59,812 INFO [Time-limited test {}] bucket.BucketCache(1441): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2d506c3-d2e6-860c-0577-0222c0ee6d4f/bucket.persistence 2023-10-04T08:06:59,812 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1040): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2023-10-04T08:06:59,812 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1040): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2023-10-04T08:06:59,812 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1040): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2023-10-04T08:06:59,812 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(50): InterruptedException in BucketCachePersister sleep interrupted 2023-10-04T08:06:59,825 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2d506c3-d2e6-860c-0577-0222c0ee6d4f/bucket.cache 2023-10-04T08:06:59,826 INFO [Time-limited test {}] bucket.BucketCache(293): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2023-10-04T08:06:59,826 INFO [Time-limited test {}] bucket.BucketAllocator(339): Cache totalSize=31518720, buckets=15, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2023-10-04T08:06:59,826 INFO [Time-limited test {}] bucket.BucketCachePersister(35): BucketCachePersister started with interval: 1000 2023-10-04T08:06:59,827 INFO [Time-limited test {}] bucket.BucketCache(340): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2d506c3-d2e6-860c-0577-0222c0ee6d4f/bucket.cache, capacity=32 MB, blockSize=8 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2d506c3-d2e6-860c-0577-0222c0ee6d4f/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2023-10-04T08:06:59,838 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestVerifyBucketCacheFile#testRetrieveFromFile[0: blockSize=8,192, bucketSizes=null] Thread=26 (was 21) - Thread LEAK? -, OpenFileDescriptor=281 (was 280) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=472 (was 472), ProcessCount=167 (was 167), AvailableMemoryMB=6636 (was 6636) 2023-10-04T08:06:59,848 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestVerifyBucketCacheFile#testModifiedBucketCacheFileData[0: blockSize=8,192, bucketSizes=null] Thread=26, OpenFileDescriptor=281, MaxFileDescriptor=60000, SystemLoadAverage=472, ProcessCount=167, AvailableMemoryMB=6636 2023-10-04T08:06:59,859 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b75dca16-01bf-e208-480f-e18a7611895e/hadoop-log-dir so I do NOT create it in target/test-data/63392740-966b-3ebf-0a78-ca557c1b94b9 2023-10-04T08:06:59,859 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.log.dir property value differs in configuration and system: Configuration=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/../logs while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b75dca16-01bf-e208-480f-e18a7611895e/hadoop-log-dir Erasing configuration value by system value. 2023-10-04T08:06:59,859 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b75dca16-01bf-e208-480f-e18a7611895e/hadoop-tmp-dir so I do NOT create it in target/test-data/63392740-966b-3ebf-0a78-ca557c1b94b9 2023-10-04T08:06:59,859 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.tmp.dir property value differs in configuration and system: Configuration=/tmp/hadoop-jenkins while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b75dca16-01bf-e208-480f-e18a7611895e/hadoop-tmp-dir Erasing configuration value by system value. 2023-10-04T08:06:59,859 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/63392740-966b-3ebf-0a78-ca557c1b94b9 2023-10-04T08:06:59,860 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-10-04T08:06:59,870 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/63392740-966b-3ebf-0a78-ca557c1b94b9/bucket.cache 2023-10-04T08:06:59,870 INFO [Time-limited test {}] bucket.BucketCache(293): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2023-10-04T08:06:59,870 INFO [Time-limited test {}] bucket.BucketAllocator(339): Cache totalSize=31518720, buckets=15, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2023-10-04T08:06:59,871 INFO [Time-limited test {}] bucket.BucketCachePersister(35): BucketCachePersister started with interval: 1000 2023-10-04T08:06:59,871 INFO [Time-limited test {}] bucket.BucketCache(340): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/63392740-966b-3ebf-0a78-ca557c1b94b9/bucket.cache, capacity=32 MB, blockSize=8 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/63392740-966b-3ebf-0a78-ca557c1b94b9/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2023-10-04T08:06:59,972 INFO [Time-limited test {}] bucket.BucketCache(1441): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/63392740-966b-3ebf-0a78-ca557c1b94b9/bucket.persistence 2023-10-04T08:06:59,972 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1040): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2023-10-04T08:06:59,972 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1040): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2023-10-04T08:06:59,972 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1040): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2023-10-04T08:06:59,973 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(50): InterruptedException in BucketCachePersister sleep interrupted 2023-10-04T08:06:59,986 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/63392740-966b-3ebf-0a78-ca557c1b94b9/bucket.cache 2023-10-04T08:06:59,987 INFO [Time-limited test {}] bucket.BucketCache(293): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2023-10-04T08:06:59,987 INFO [Time-limited test {}] bucket.BucketAllocator(339): Cache totalSize=31518720, buckets=15, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2023-10-04T08:06:59,987 INFO [Time-limited test {}] bucket.BucketCachePersister(35): BucketCachePersister started with interval: 1000 2023-10-04T08:06:59,989 WARN [Time-limited test {}] bucket.BucketCache(1372): Checksum for cache file failed. We need to validate each cache key in the backing map. This may take some time... 2023-10-04T08:06:59,989 DEBUG [Time-limited test {}] bucket.BucketCache(1380): Check for key 1900409212605002326_0 failed. Removing it from map. 2023-10-04T08:06:59,990 INFO [Time-limited test {}] bucket.BucketCache(1385): Finished validating 1 keys in the backing map. Recovered: 0. This took 0ms. 2023-10-04T08:06:59,990 INFO [Time-limited test {}] bucket.BucketAllocator(339): Cache totalSize=31518720, buckets=15, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2023-10-04T08:06:59,990 INFO [Time-limited test {}] bucket.BucketCache(340): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/63392740-966b-3ebf-0a78-ca557c1b94b9/bucket.cache, capacity=32 MB, blockSize=8 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/63392740-966b-3ebf-0a78-ca557c1b94b9/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2023-10-04T08:06:59,998 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestVerifyBucketCacheFile#testModifiedBucketCacheFileData[0: blockSize=8,192, bucketSizes=null] Thread=31 (was 26) - Thread LEAK? -, OpenFileDescriptor=282 (was 281) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=472 (was 472), ProcessCount=167 (was 167), AvailableMemoryMB=6635 (was 6636) 2023-10-04T08:07:00,006 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestVerifyBucketCacheFile#testModifiedBucketCacheFileTime[0: blockSize=8,192, bucketSizes=null] Thread=31, OpenFileDescriptor=282, MaxFileDescriptor=60000, SystemLoadAverage=472, ProcessCount=167, AvailableMemoryMB=6634 2023-10-04T08:07:00,017 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b75dca16-01bf-e208-480f-e18a7611895e/hadoop-log-dir so I do NOT create it in target/test-data/58c1946f-0224-c2d5-d3b4-291ea1e82ada 2023-10-04T08:07:00,017 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.log.dir property value differs in configuration and system: Configuration=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/../logs while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b75dca16-01bf-e208-480f-e18a7611895e/hadoop-log-dir Erasing configuration value by system value. 2023-10-04T08:07:00,017 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b75dca16-01bf-e208-480f-e18a7611895e/hadoop-tmp-dir so I do NOT create it in target/test-data/58c1946f-0224-c2d5-d3b4-291ea1e82ada 2023-10-04T08:07:00,017 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.tmp.dir property value differs in configuration and system: Configuration=/tmp/hadoop-jenkins while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b75dca16-01bf-e208-480f-e18a7611895e/hadoop-tmp-dir Erasing configuration value by system value. 2023-10-04T08:07:00,017 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/58c1946f-0224-c2d5-d3b4-291ea1e82ada 2023-10-04T08:07:00,018 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-10-04T08:07:00,028 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/58c1946f-0224-c2d5-d3b4-291ea1e82ada/bucket.cache 2023-10-04T08:07:00,028 INFO [Time-limited test {}] bucket.BucketCache(293): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2023-10-04T08:07:00,028 INFO [Time-limited test {}] bucket.BucketAllocator(339): Cache totalSize=31518720, buckets=15, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2023-10-04T08:07:00,029 INFO [Time-limited test {}] bucket.BucketCachePersister(35): BucketCachePersister started with interval: 1000 2023-10-04T08:07:00,029 INFO [Time-limited test {}] bucket.BucketCache(340): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/58c1946f-0224-c2d5-d3b4-291ea1e82ada/bucket.cache, capacity=32 MB, blockSize=8 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/58c1946f-0224-c2d5-d3b4-291ea1e82ada/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2023-10-04T08:07:00,130 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1040): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2023-10-04T08:07:00,130 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1040): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2023-10-04T08:07:00,130 INFO [Time-limited test {}] bucket.BucketCache(1441): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/58c1946f-0224-c2d5-d3b4-291ea1e82ada/bucket.persistence 2023-10-04T08:07:00,130 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1040): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2023-10-04T08:07:00,130 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(50): InterruptedException in BucketCachePersister sleep interrupted 2023-10-04T08:07:00,145 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/58c1946f-0224-c2d5-d3b4-291ea1e82ada/bucket.cache 2023-10-04T08:07:00,145 INFO [Time-limited test {}] bucket.BucketCache(293): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2023-10-04T08:07:00,145 INFO [Time-limited test {}] bucket.BucketAllocator(339): Cache totalSize=31518720, buckets=15, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2023-10-04T08:07:00,145 INFO [Time-limited test {}] bucket.BucketCachePersister(35): BucketCachePersister started with interval: 1000 2023-10-04T08:07:00,148 WARN [Time-limited test {}] bucket.BucketCache(1372): Checksum for cache file failed. We need to validate each cache key in the backing map. This may take some time... 2023-10-04T08:07:00,148 INFO [Time-limited test {}] bucket.BucketCache(1385): Finished validating 1 keys in the backing map. Recovered: 1. This took 0ms. 2023-10-04T08:07:00,148 INFO [Time-limited test {}] bucket.BucketAllocator(339): Cache totalSize=31518720, buckets=15, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2023-10-04T08:07:00,148 INFO [Time-limited test {}] bucket.BucketCache(340): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/58c1946f-0224-c2d5-d3b4-291ea1e82ada/bucket.cache, capacity=32 MB, blockSize=8 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/58c1946f-0224-c2d5-d3b4-291ea1e82ada/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2023-10-04T08:07:00,157 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestVerifyBucketCacheFile#testModifiedBucketCacheFileTime[0: blockSize=8,192, bucketSizes=null] Thread=36 (was 31) - Thread LEAK? -, OpenFileDescriptor=283 (was 282) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=472 (was 472), ProcessCount=167 (was 167), AvailableMemoryMB=6634 (was 6634) 2023-10-04T08:07:00,164 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestVerifyBucketCacheFile#testRetrieveFromFileAfterDelete[0: blockSize=8,192, bucketSizes=null] Thread=36, OpenFileDescriptor=283, MaxFileDescriptor=60000, SystemLoadAverage=472, ProcessCount=167, AvailableMemoryMB=6634 2023-10-04T08:07:00,175 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b75dca16-01bf-e208-480f-e18a7611895e/hadoop-log-dir so I do NOT create it in target/test-data/e7e61480-2da8-4df0-f3c6-ff5373da1988 2023-10-04T08:07:00,175 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.log.dir property value differs in configuration and system: Configuration=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/../logs while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b75dca16-01bf-e208-480f-e18a7611895e/hadoop-log-dir Erasing configuration value by system value. 2023-10-04T08:07:00,175 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b75dca16-01bf-e208-480f-e18a7611895e/hadoop-tmp-dir so I do NOT create it in target/test-data/e7e61480-2da8-4df0-f3c6-ff5373da1988 2023-10-04T08:07:00,175 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.tmp.dir property value differs in configuration and system: Configuration=/tmp/hadoop-jenkins while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b75dca16-01bf-e208-480f-e18a7611895e/hadoop-tmp-dir Erasing configuration value by system value. 2023-10-04T08:07:00,176 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/e7e61480-2da8-4df0-f3c6-ff5373da1988 2023-10-04T08:07:00,176 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-10-04T08:07:00,177 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/e7e61480-2da8-4df0-f3c6-ff5373da1988/bucket.cache 2023-10-04T08:07:00,177 INFO [Time-limited test {}] bucket.BucketCache(293): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2023-10-04T08:07:00,177 INFO [Time-limited test {}] bucket.BucketAllocator(339): Cache totalSize=31518720, buckets=15, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2023-10-04T08:07:00,177 INFO [Time-limited test {}] bucket.BucketCachePersister(35): BucketCachePersister started with interval: 300 2023-10-04T08:07:00,178 INFO [Time-limited test {}] bucket.BucketCache(340): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/e7e61480-2da8-4df0-f3c6-ff5373da1988/bucket.cache, capacity=32 MB, blockSize=8 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/e7e61480-2da8-4df0-f3c6-ff5373da1988/bucket.persistence1696406820176, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2023-10-04T08:07:00,279 INFO [Time-limited test {}] bucket.BucketCache(1441): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/e7e61480-2da8-4df0-f3c6-ff5373da1988/bucket.persistence1696406820176 2023-10-04T08:07:00,279 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1040): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2023-10-04T08:07:00,279 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1040): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2023-10-04T08:07:00,279 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1040): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2023-10-04T08:07:00,279 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(50): InterruptedException in BucketCachePersister sleep interrupted 2023-10-04T08:07:00,632 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/e7e61480-2da8-4df0-f3c6-ff5373da1988/bucket.cache 2023-10-04T08:07:00,633 INFO [Time-limited test {}] bucket.BucketCache(293): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2023-10-04T08:07:00,633 INFO [Time-limited test {}] bucket.BucketAllocator(339): Cache totalSize=31518720, buckets=15, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2023-10-04T08:07:00,633 INFO [Time-limited test {}] bucket.BucketCachePersister(35): BucketCachePersister started with interval: 300 2023-10-04T08:07:00,634 INFO [Time-limited test {}] bucket.BucketCache(340): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/e7e61480-2da8-4df0-f3c6-ff5373da1988/bucket.cache, capacity=32 MB, blockSize=8 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/e7e61480-2da8-4df0-f3c6-ff5373da1988/bucket.persistence1696406820176, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2023-10-04T08:07:00,642 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestVerifyBucketCacheFile#testRetrieveFromFileAfterDelete[0: blockSize=8,192, bucketSizes=null] Thread=41 (was 36) - Thread LEAK? -, OpenFileDescriptor=284 (was 283) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=472 (was 472), ProcessCount=167 (was 167), AvailableMemoryMB=6633 (was 6634) 2023-10-04T08:07:00,650 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestVerifyBucketCacheFile#testBucketCacheRecovery[1: blockSize=16,384, bucketSizes=[I@82de64a] Thread=41, OpenFileDescriptor=284, MaxFileDescriptor=60000, SystemLoadAverage=472, ProcessCount=167, AvailableMemoryMB=6633 2023-10-04T08:07:00,662 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b75dca16-01bf-e208-480f-e18a7611895e/hadoop-log-dir so I do NOT create it in target/test-data/d3413e3d-f1a5-7ddc-a91c-166d2176ce01 2023-10-04T08:07:00,662 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.log.dir property value differs in configuration and system: Configuration=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/../logs while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b75dca16-01bf-e208-480f-e18a7611895e/hadoop-log-dir Erasing configuration value by system value. 2023-10-04T08:07:00,662 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b75dca16-01bf-e208-480f-e18a7611895e/hadoop-tmp-dir so I do NOT create it in target/test-data/d3413e3d-f1a5-7ddc-a91c-166d2176ce01 2023-10-04T08:07:00,663 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.tmp.dir property value differs in configuration and system: Configuration=/tmp/hadoop-jenkins while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b75dca16-01bf-e208-480f-e18a7611895e/hadoop-tmp-dir Erasing configuration value by system value. 2023-10-04T08:07:00,663 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d3413e3d-f1a5-7ddc-a91c-166d2176ce01 2023-10-04T08:07:00,663 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-10-04T08:07:00,674 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d3413e3d-f1a5-7ddc-a91c-166d2176ce01/bucket.cache 2023-10-04T08:07:00,674 INFO [Time-limited test {}] bucket.BucketCache(293): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2023-10-04T08:07:00,674 INFO [Time-limited test {}] bucket.BucketAllocator(339): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2023-10-04T08:07:00,675 INFO [Time-limited test {}] bucket.BucketCachePersister(35): BucketCachePersister started with interval: 9223372036854775807 2023-10-04T08:07:00,675 INFO [Time-limited test {}] bucket.BucketCache(340): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d3413e3d-f1a5-7ddc-a91c-166d2176ce01/bucket.cache, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d3413e3d-f1a5-7ddc-a91c-166d2176ce01/bucket.persistence1696406820674, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2023-10-04T08:07:00,827 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(43): Cache is inconsistent, persisting to disk 2023-10-04T08:07:00,827 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(48): IOException in BucketCachePersister /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2d506c3-d2e6-860c-0577-0222c0ee6d4f/bucket.persistence1696406820827 (No such file or directory) 2023-10-04T08:07:00,934 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(43): Cache is inconsistent, persisting to disk 2023-10-04T08:07:00,987 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(43): Cache is inconsistent, persisting to disk 2023-10-04T08:07:00,988 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(48): IOException in BucketCachePersister /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/63392740-966b-3ebf-0a78-ca557c1b94b9/bucket.persistence1696406820987 (No such file or directory) 2023-10-04T08:07:01,079 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d3413e3d-f1a5-7ddc-a91c-166d2176ce01/bucket.cache 2023-10-04T08:07:01,080 INFO [Time-limited test {}] bucket.BucketCache(293): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2023-10-04T08:07:01,080 INFO [Time-limited test {}] bucket.BucketAllocator(339): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2023-10-04T08:07:01,080 INFO [Time-limited test {}] bucket.BucketCachePersister(35): BucketCachePersister started with interval: 9223372036854775807 2023-10-04T08:07:01,082 WARN [Time-limited test {}] bucket.BucketCache(1372): Checksum for cache file failed. We need to validate each cache key in the backing map. This may take some time... 2023-10-04T08:07:01,083 DEBUG [Time-limited test {}] bucket.BucketCache(1380): Check for key 5856376773487661839_0 failed. Removing it from map. 2023-10-04T08:07:01,083 INFO [Time-limited test {}] bucket.BucketCache(1385): Finished validating 3 keys in the backing map. Recovered: 2. This took 0ms. 2023-10-04T08:07:01,083 INFO [Time-limited test {}] bucket.BucketAllocator(339): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2023-10-04T08:07:01,084 INFO [Time-limited test {}] bucket.BucketCache(340): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d3413e3d-f1a5-7ddc-a91c-166d2176ce01/bucket.cache, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d3413e3d-f1a5-7ddc-a91c-166d2176ce01/bucket.persistence1696406820674, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2023-10-04T08:07:01,093 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestVerifyBucketCacheFile#testBucketCacheRecovery[1: blockSize=16,384, bucketSizes=[I@82de64a] Thread=51 (was 41) - Thread LEAK? -, OpenFileDescriptor=286 (was 284) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=472 (was 472), ProcessCount=167 (was 167), AvailableMemoryMB=6630 (was 6633) 2023-10-04T08:07:01,101 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestVerifyBucketCacheFile#testRetrieveFromFile[1: blockSize=16,384, bucketSizes=[I@82de64a] Thread=51, OpenFileDescriptor=286, MaxFileDescriptor=60000, SystemLoadAverage=472, ProcessCount=167, AvailableMemoryMB=6630 2023-10-04T08:07:01,113 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b75dca16-01bf-e208-480f-e18a7611895e/hadoop-log-dir so I do NOT create it in target/test-data/36b6b0c1-2ded-d320-a2b0-e232a63b1f45 2023-10-04T08:07:01,113 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.log.dir property value differs in configuration and system: Configuration=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/../logs while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b75dca16-01bf-e208-480f-e18a7611895e/hadoop-log-dir Erasing configuration value by system value. 2023-10-04T08:07:01,113 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b75dca16-01bf-e208-480f-e18a7611895e/hadoop-tmp-dir so I do NOT create it in target/test-data/36b6b0c1-2ded-d320-a2b0-e232a63b1f45 2023-10-04T08:07:01,113 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.tmp.dir property value differs in configuration and system: Configuration=/tmp/hadoop-jenkins while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b75dca16-01bf-e208-480f-e18a7611895e/hadoop-tmp-dir Erasing configuration value by system value. 2023-10-04T08:07:01,113 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/36b6b0c1-2ded-d320-a2b0-e232a63b1f45 2023-10-04T08:07:01,113 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-10-04T08:07:01,124 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/36b6b0c1-2ded-d320-a2b0-e232a63b1f45/bucket.cache 2023-10-04T08:07:01,124 INFO [Time-limited test {}] bucket.BucketCache(293): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2023-10-04T08:07:01,124 INFO [Time-limited test {}] bucket.BucketAllocator(339): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2023-10-04T08:07:01,124 INFO [Time-limited test {}] bucket.BucketCachePersister(35): BucketCachePersister started with interval: 1000 2023-10-04T08:07:01,125 INFO [Time-limited test {}] bucket.BucketCache(340): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/36b6b0c1-2ded-d320-a2b0-e232a63b1f45/bucket.cache, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/36b6b0c1-2ded-d320-a2b0-e232a63b1f45/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2023-10-04T08:07:01,146 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(43): Cache is inconsistent, persisting to disk 2023-10-04T08:07:01,146 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(48): IOException in BucketCachePersister /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/58c1946f-0224-c2d5-d3b4-291ea1e82ada/bucket.persistence1696406821146 (No such file or directory) 2023-10-04T08:07:01,225 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1040): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2023-10-04T08:07:01,225 INFO [Time-limited test {}] bucket.BucketCache(1441): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/36b6b0c1-2ded-d320-a2b0-e232a63b1f45/bucket.persistence 2023-10-04T08:07:01,225 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1040): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2023-10-04T08:07:01,225 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1040): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2023-10-04T08:07:01,226 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(50): InterruptedException in BucketCachePersister sleep interrupted 2023-10-04T08:07:01,236 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(43): Cache is inconsistent, persisting to disk 2023-10-04T08:07:01,239 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/36b6b0c1-2ded-d320-a2b0-e232a63b1f45/bucket.cache 2023-10-04T08:07:01,239 INFO [Time-limited test {}] bucket.BucketCache(293): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2023-10-04T08:07:01,239 INFO [Time-limited test {}] bucket.BucketAllocator(339): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2023-10-04T08:07:01,239 INFO [Time-limited test {}] bucket.BucketCachePersister(35): BucketCachePersister started with interval: 1000 2023-10-04T08:07:01,241 INFO [Time-limited test {}] bucket.BucketAllocator(339): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2023-10-04T08:07:01,242 INFO [Time-limited test {}] bucket.BucketCache(340): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/36b6b0c1-2ded-d320-a2b0-e232a63b1f45/bucket.cache, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/36b6b0c1-2ded-d320-a2b0-e232a63b1f45/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2023-10-04T08:07:01,242 INFO [Time-limited test {}] bucket.BucketCache(1441): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/36b6b0c1-2ded-d320-a2b0-e232a63b1f45/bucket.persistence 2023-10-04T08:07:01,242 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1040): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2023-10-04T08:07:01,242 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1040): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2023-10-04T08:07:01,242 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1040): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2023-10-04T08:07:01,242 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(50): InterruptedException in BucketCachePersister sleep interrupted 2023-10-04T08:07:01,255 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/36b6b0c1-2ded-d320-a2b0-e232a63b1f45/bucket.cache 2023-10-04T08:07:01,255 INFO [Time-limited test {}] bucket.BucketCache(293): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2023-10-04T08:07:01,256 INFO [Time-limited test {}] bucket.BucketAllocator(339): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2023-10-04T08:07:01,256 INFO [Time-limited test {}] bucket.BucketCachePersister(35): BucketCachePersister started with interval: 1000 2023-10-04T08:07:01,257 WARN [Time-limited test {}] bucket.BucketCache(1372): Checksum for cache file failed. We need to validate each cache key in the backing map. This may take some time... 2023-10-04T08:07:01,258 DEBUG [Time-limited test {}] bucket.BucketCache(1380): Check for key -1440098253687416497_0 failed. Removing it from map. 2023-10-04T08:07:01,258 INFO [Time-limited test {}] bucket.BucketCache(1385): Finished validating 1 keys in the backing map. Recovered: 0. This took 0ms. 2023-10-04T08:07:01,258 INFO [Time-limited test {}] bucket.BucketAllocator(339): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2023-10-04T08:07:01,259 INFO [Time-limited test {}] bucket.BucketCache(340): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/36b6b0c1-2ded-d320-a2b0-e232a63b1f45/bucket.cache, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/36b6b0c1-2ded-d320-a2b0-e232a63b1f45/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2023-10-04T08:07:01,359 INFO [Time-limited test {}] bucket.BucketCache(1441): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/36b6b0c1-2ded-d320-a2b0-e232a63b1f45/bucket.persistence 2023-10-04T08:07:01,359 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1040): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2023-10-04T08:07:01,359 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1040): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2023-10-04T08:07:01,359 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1040): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2023-10-04T08:07:01,359 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(50): InterruptedException in BucketCachePersister sleep interrupted 2023-10-04T08:07:01,374 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/36b6b0c1-2ded-d320-a2b0-e232a63b1f45/bucket.cache 2023-10-04T08:07:01,374 INFO [Time-limited test {}] bucket.BucketCache(293): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2023-10-04T08:07:01,374 INFO [Time-limited test {}] bucket.BucketAllocator(339): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2023-10-04T08:07:01,375 INFO [Time-limited test {}] bucket.BucketCachePersister(35): BucketCachePersister started with interval: 1000 2023-10-04T08:07:01,375 INFO [Time-limited test {}] bucket.BucketCache(340): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/36b6b0c1-2ded-d320-a2b0-e232a63b1f45/bucket.cache, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/36b6b0c1-2ded-d320-a2b0-e232a63b1f45/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2023-10-04T08:07:01,384 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestVerifyBucketCacheFile#testRetrieveFromFile[1: blockSize=16,384, bucketSizes=[I@82de64a] Thread=56 (was 51) - Thread LEAK? -, OpenFileDescriptor=287 (was 286) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=472 (was 472), ProcessCount=167 (was 167), AvailableMemoryMB=6626 (was 6630) 2023-10-04T08:07:01,392 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestVerifyBucketCacheFile#testModifiedBucketCacheFileData[1: blockSize=16,384, bucketSizes=[I@82de64a] Thread=56, OpenFileDescriptor=287, MaxFileDescriptor=60000, SystemLoadAverage=472, ProcessCount=167, AvailableMemoryMB=6626 2023-10-04T08:07:01,403 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b75dca16-01bf-e208-480f-e18a7611895e/hadoop-log-dir so I do NOT create it in target/test-data/aabf5385-f993-4c91-d7b2-5f1e4c484e82 2023-10-04T08:07:01,403 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.log.dir property value differs in configuration and system: Configuration=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/../logs while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b75dca16-01bf-e208-480f-e18a7611895e/hadoop-log-dir Erasing configuration value by system value. 2023-10-04T08:07:01,403 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b75dca16-01bf-e208-480f-e18a7611895e/hadoop-tmp-dir so I do NOT create it in target/test-data/aabf5385-f993-4c91-d7b2-5f1e4c484e82 2023-10-04T08:07:01,403 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.tmp.dir property value differs in configuration and system: Configuration=/tmp/hadoop-jenkins while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b75dca16-01bf-e208-480f-e18a7611895e/hadoop-tmp-dir Erasing configuration value by system value. 2023-10-04T08:07:01,403 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/aabf5385-f993-4c91-d7b2-5f1e4c484e82 2023-10-04T08:07:01,403 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-10-04T08:07:01,414 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/aabf5385-f993-4c91-d7b2-5f1e4c484e82/bucket.cache 2023-10-04T08:07:01,414 INFO [Time-limited test {}] bucket.BucketCache(293): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2023-10-04T08:07:01,414 INFO [Time-limited test {}] bucket.BucketAllocator(339): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2023-10-04T08:07:01,414 INFO [Time-limited test {}] bucket.BucketCachePersister(35): BucketCachePersister started with interval: 1000 2023-10-04T08:07:01,415 INFO [Time-limited test {}] bucket.BucketCache(340): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/aabf5385-f993-4c91-d7b2-5f1e4c484e82/bucket.cache, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/aabf5385-f993-4c91-d7b2-5f1e4c484e82/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2023-10-04T08:07:01,516 INFO [Time-limited test {}] bucket.BucketCache(1441): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/aabf5385-f993-4c91-d7b2-5f1e4c484e82/bucket.persistence 2023-10-04T08:07:01,516 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1040): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2023-10-04T08:07:01,516 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1040): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2023-10-04T08:07:01,516 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1040): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2023-10-04T08:07:01,516 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(50): InterruptedException in BucketCachePersister sleep interrupted 2023-10-04T08:07:01,531 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/aabf5385-f993-4c91-d7b2-5f1e4c484e82/bucket.cache 2023-10-04T08:07:01,531 INFO [Time-limited test {}] bucket.BucketCache(293): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2023-10-04T08:07:01,531 INFO [Time-limited test {}] bucket.BucketAllocator(339): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2023-10-04T08:07:01,531 INFO [Time-limited test {}] bucket.BucketCachePersister(35): BucketCachePersister started with interval: 1000 2023-10-04T08:07:01,534 WARN [Time-limited test {}] bucket.BucketCache(1372): Checksum for cache file failed. We need to validate each cache key in the backing map. This may take some time... 2023-10-04T08:07:01,534 DEBUG [Time-limited test {}] bucket.BucketCache(1380): Check for key 4422503229656182151_0 failed. Removing it from map. 2023-10-04T08:07:01,534 INFO [Time-limited test {}] bucket.BucketCache(1385): Finished validating 1 keys in the backing map. Recovered: 0. This took 0ms. 2023-10-04T08:07:01,534 INFO [Time-limited test {}] bucket.BucketAllocator(339): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2023-10-04T08:07:01,535 INFO [Time-limited test {}] bucket.BucketCache(340): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/aabf5385-f993-4c91-d7b2-5f1e4c484e82/bucket.cache, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/aabf5385-f993-4c91-d7b2-5f1e4c484e82/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2023-10-04T08:07:01,538 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(43): Cache is inconsistent, persisting to disk 2023-10-04T08:07:01,545 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestVerifyBucketCacheFile#testModifiedBucketCacheFileData[1: blockSize=16,384, bucketSizes=[I@82de64a] Thread=61 (was 56) - Thread LEAK? -, OpenFileDescriptor=288 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=472 (was 472), ProcessCount=167 (was 167), AvailableMemoryMB=6620 (was 6626) 2023-10-04T08:07:01,553 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestVerifyBucketCacheFile#testModifiedBucketCacheFileTime[1: blockSize=16,384, bucketSizes=[I@82de64a] Thread=62, OpenFileDescriptor=288, MaxFileDescriptor=60000, SystemLoadAverage=472, ProcessCount=167, AvailableMemoryMB=6620 2023-10-04T08:07:01,564 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b75dca16-01bf-e208-480f-e18a7611895e/hadoop-log-dir so I do NOT create it in target/test-data/36a88c5d-8074-9e64-d587-cc1001be2bda 2023-10-04T08:07:01,565 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.log.dir property value differs in configuration and system: Configuration=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/../logs while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b75dca16-01bf-e208-480f-e18a7611895e/hadoop-log-dir Erasing configuration value by system value. 2023-10-04T08:07:01,565 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b75dca16-01bf-e208-480f-e18a7611895e/hadoop-tmp-dir so I do NOT create it in target/test-data/36a88c5d-8074-9e64-d587-cc1001be2bda 2023-10-04T08:07:01,565 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.tmp.dir property value differs in configuration and system: Configuration=/tmp/hadoop-jenkins while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b75dca16-01bf-e208-480f-e18a7611895e/hadoop-tmp-dir Erasing configuration value by system value. 2023-10-04T08:07:01,565 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/36a88c5d-8074-9e64-d587-cc1001be2bda 2023-10-04T08:07:01,565 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-10-04T08:07:01,582 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/36a88c5d-8074-9e64-d587-cc1001be2bda/bucket.cache 2023-10-04T08:07:01,582 INFO [Time-limited test {}] bucket.BucketCache(293): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2023-10-04T08:07:01,582 INFO [Time-limited test {}] bucket.BucketAllocator(339): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2023-10-04T08:07:01,582 INFO [Time-limited test {}] bucket.BucketCachePersister(35): BucketCachePersister started with interval: 1000 2023-10-04T08:07:01,583 INFO [Time-limited test {}] bucket.BucketCache(340): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/36a88c5d-8074-9e64-d587-cc1001be2bda/bucket.cache, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/36a88c5d-8074-9e64-d587-cc1001be2bda/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2023-10-04T08:07:01,684 INFO [Time-limited test {}] bucket.BucketCache(1441): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/36a88c5d-8074-9e64-d587-cc1001be2bda/bucket.persistence 2023-10-04T08:07:01,684 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1040): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2023-10-04T08:07:01,684 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1040): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2023-10-04T08:07:01,684 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1040): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2023-10-04T08:07:01,685 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(50): InterruptedException in BucketCachePersister sleep interrupted 2023-10-04T08:07:01,700 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/36a88c5d-8074-9e64-d587-cc1001be2bda/bucket.cache 2023-10-04T08:07:01,700 INFO [Time-limited test {}] bucket.BucketCache(293): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2023-10-04T08:07:01,700 INFO [Time-limited test {}] bucket.BucketAllocator(339): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2023-10-04T08:07:01,700 INFO [Time-limited test {}] bucket.BucketCachePersister(35): BucketCachePersister started with interval: 1000 2023-10-04T08:07:01,703 WARN [Time-limited test {}] bucket.BucketCache(1372): Checksum for cache file failed. We need to validate each cache key in the backing map. This may take some time... 2023-10-04T08:07:01,703 INFO [Time-limited test {}] bucket.BucketCache(1385): Finished validating 1 keys in the backing map. Recovered: 1. This took 0ms. 2023-10-04T08:07:01,703 INFO [Time-limited test {}] bucket.BucketAllocator(339): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2023-10-04T08:07:01,704 INFO [Time-limited test {}] bucket.BucketCache(340): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/36a88c5d-8074-9e64-d587-cc1001be2bda/bucket.cache, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/36a88c5d-8074-9e64-d587-cc1001be2bda/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2023-10-04T08:07:01,712 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestVerifyBucketCacheFile#testModifiedBucketCacheFileTime[1: blockSize=16,384, bucketSizes=[I@82de64a] Thread=67 (was 62) - Thread LEAK? -, OpenFileDescriptor=289 (was 288) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=472 (was 472), ProcessCount=167 (was 167), AvailableMemoryMB=6614 (was 6620) 2023-10-04T08:07:01,720 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.bucket.TestVerifyBucketCacheFile#testRetrieveFromFileAfterDelete[1: blockSize=16,384, bucketSizes=[I@82de64a] Thread=67, OpenFileDescriptor=289, MaxFileDescriptor=60000, SystemLoadAverage=472, ProcessCount=167, AvailableMemoryMB=6614 2023-10-04T08:07:01,732 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b75dca16-01bf-e208-480f-e18a7611895e/hadoop-log-dir so I do NOT create it in target/test-data/85d174e2-076c-91f2-5e4f-044d13b77db5 2023-10-04T08:07:01,732 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.log.dir property value differs in configuration and system: Configuration=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/../logs while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b75dca16-01bf-e208-480f-e18a7611895e/hadoop-log-dir Erasing configuration value by system value. 2023-10-04T08:07:01,732 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b75dca16-01bf-e208-480f-e18a7611895e/hadoop-tmp-dir so I do NOT create it in target/test-data/85d174e2-076c-91f2-5e4f-044d13b77db5 2023-10-04T08:07:01,732 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.tmp.dir property value differs in configuration and system: Configuration=/tmp/hadoop-jenkins while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/b75dca16-01bf-e208-480f-e18a7611895e/hadoop-tmp-dir Erasing configuration value by system value. 2023-10-04T08:07:01,732 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/85d174e2-076c-91f2-5e4f-044d13b77db5 2023-10-04T08:07:01,732 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-10-04T08:07:01,733 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/85d174e2-076c-91f2-5e4f-044d13b77db5/bucket.cache 2023-10-04T08:07:01,733 INFO [Time-limited test {}] bucket.BucketCache(293): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2023-10-04T08:07:01,733 INFO [Time-limited test {}] bucket.BucketAllocator(339): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2023-10-04T08:07:01,733 INFO [Time-limited test {}] bucket.BucketCachePersister(35): BucketCachePersister started with interval: 300 2023-10-04T08:07:01,736 INFO [Time-limited test {}] bucket.BucketCache(340): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/85d174e2-076c-91f2-5e4f-044d13b77db5/bucket.cache, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/85d174e2-076c-91f2-5e4f-044d13b77db5/bucket.persistence1696406821733, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2023-10-04T08:07:01,827 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(43): Cache is inconsistent, persisting to disk 2023-10-04T08:07:01,827 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(48): IOException in BucketCachePersister /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/d2d506c3-d2e6-860c-0577-0222c0ee6d4f/bucket.persistence1696406821827 (No such file or directory) 2023-10-04T08:07:01,836 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1040): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2023-10-04T08:07:01,836 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1040): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2023-10-04T08:07:01,836 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1040): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2023-10-04T08:07:01,836 INFO [Time-limited test {}] bucket.BucketCache(1441): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/85d174e2-076c-91f2-5e4f-044d13b77db5/bucket.persistence1696406821733 2023-10-04T08:07:01,837 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(50): InterruptedException in BucketCachePersister sleep interrupted 2023-10-04T08:07:01,841 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(43): Cache is inconsistent, persisting to disk 2023-10-04T08:07:02,190 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 32 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/85d174e2-076c-91f2-5e4f-044d13b77db5/bucket.cache 2023-10-04T08:07:02,191 INFO [Time-limited test {}] bucket.BucketCache(293): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25 2023-10-04T08:07:02,191 INFO [Time-limited test {}] bucket.BucketAllocator(339): Cache totalSize=33288192, buckets=63, bucket capacity=528384=(4*132096)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2023-10-04T08:07:02,191 INFO [Time-limited test {}] bucket.BucketCachePersister(35): BucketCachePersister started with interval: 300 2023-10-04T08:07:02,192 INFO [Time-limited test {}] bucket.BucketCache(340): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/85d174e2-076c-91f2-5e4f-044d13b77db5/bucket.cache, capacity=32 MB, blockSize=16 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/85d174e2-076c-91f2-5e4f-044d13b77db5/bucket.persistence1696406821733, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2023-10-04T08:07:02,201 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.bucket.TestVerifyBucketCacheFile#testRetrieveFromFileAfterDelete[1: blockSize=16,384, bucketSizes=[I@82de64a] Thread=72 (was 67) - Thread LEAK? -, OpenFileDescriptor=290 (was 289) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=472 (was 472), ProcessCount=167 (was 167), AvailableMemoryMB=6612 (was 6614)