2024-12-07 21:57:55,128 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-12-07 21:57:55,150 main DEBUG Took 0.018115 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-07 21:57:55,151 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-07 21:57:55,151 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-07 21:57:55,172 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-07 21:57:55,174 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 21:57:55,184 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-07 21:57:55,204 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 21:57:55,207 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 21:57:55,208 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 21:57:55,208 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 21:57:55,209 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 21:57:55,209 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 21:57:55,211 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 21:57:55,211 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 21:57:55,212 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 21:57:55,213 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 21:57:55,214 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 21:57:55,214 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 21:57:55,215 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 21:57:55,215 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 21:57:55,216 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 21:57:55,217 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 21:57:55,217 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 21:57:55,218 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 21:57:55,218 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 21:57:55,219 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 21:57:55,219 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 21:57:55,220 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 21:57:55,220 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 21:57:55,221 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 21:57:55,221 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 21:57:55,222 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-07 21:57:55,226 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 21:57:55,228 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-07 21:57:55,231 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-07 21:57:55,231 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-07 21:57:55,233 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-07 21:57:55,234 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-07 21:57:55,245 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-07 21:57:55,251 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-07 21:57:55,254 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-07 21:57:55,254 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-07 21:57:55,255 main DEBUG createAppenders(={Console}) 2024-12-07 21:57:55,256 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-12-07 21:57:55,256 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-12-07 21:57:55,257 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-12-07 21:57:55,258 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-07 21:57:55,258 main DEBUG OutputStream closed 2024-12-07 21:57:55,259 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-07 21:57:55,259 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-07 21:57:55,259 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-12-07 21:57:55,369 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-07 21:57:55,375 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-07 21:57:55,377 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-07 21:57:55,378 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-07 21:57:55,379 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-07 21:57:55,379 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-07 21:57:55,380 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-07 21:57:55,380 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-07 21:57:55,380 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-07 21:57:55,381 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-07 21:57:55,381 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-07 21:57:55,381 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-07 21:57:55,382 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-07 21:57:55,382 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-07 21:57:55,382 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-07 21:57:55,383 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-07 21:57:55,383 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-07 21:57:55,384 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-07 21:57:55,387 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-07 21:57:55,387 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-12-07 21:57:55,387 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-07 21:57:55,388 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-12-07T21:57:55,414 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-12-07 21:57:55,423 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-07 21:57:55,424 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-07T21:57:55,757 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38 2024-12-07T21:57:55,802 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=427, ProcessCount=11, AvailableMemoryMB=18422 2024-12-07T21:57:55,984 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-07T21:57:56,154 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T21:57:56,219 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/cluster_51f78024-177f-f82b-0167-701c52cc1a8a, deleteOnExit=true 2024-12-07T21:57:56,379 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/cluster_51f78024-177f-f82b-0167-701c52cc1a8a/zookeeper_0, clientPort=60762, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/cluster_51f78024-177f-f82b-0167-701c52cc1a8a/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/cluster_51f78024-177f-f82b-0167-701c52cc1a8a/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-07T21:57:56,393 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60762 2024-12-07T21:57:56,393 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T21:57:56,505 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442 with version=8 2024-12-07T21:57:56,506 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/hbase-staging 2024-12-07T21:57:56,609 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-07T21:57:56,907 INFO [Time-limited test {}] client.ConnectionUtils(128): master/dbf4292a68c8:0 server-side Connection retries=45 2024-12-07T21:57:56,919 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T21:57:56,920 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-07T21:57:56,931 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-07T21:57:56,931 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T21:57:56,932 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-07T21:57:57,085 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-07T21:57:57,185 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-07T21:57:57,226 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-07T21:57:57,235 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-07T21:57:57,238 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-07T21:57:57,281 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 110259 (auto-detected) 2024-12-07T21:57:57,282 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-07T21:57:57,300 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38375 2024-12-07T21:57:57,321 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38375 connecting to ZooKeeper ensemble=127.0.0.1:60762 2024-12-07T21:57:57,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:383750x0, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-07T21:57:57,453 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38375-0x1000256da950000 connected 2024-12-07T21:57:57,550 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T21:57:57,552 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T21:57:57,561 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38375-0x1000256da950000, quorum=127.0.0.1:60762, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T21:57:57,565 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442, hbase.cluster.distributed=false 2024-12-07T21:57:57,585 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38375-0x1000256da950000, quorum=127.0.0.1:60762, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-07T21:57:57,591 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38375 2024-12-07T21:57:57,592 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38375 2024-12-07T21:57:57,592 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38375 2024-12-07T21:57:57,592 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38375 2024-12-07T21:57:57,593 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38375 2024-12-07T21:57:57,665 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/dbf4292a68c8:0 server-side Connection retries=45 2024-12-07T21:57:57,667 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T21:57:57,667 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-07T21:57:57,667 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-07T21:57:57,667 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T21:57:57,668 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-07T21:57:57,670 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-07T21:57:57,674 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-07T21:57:57,675 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46761 2024-12-07T21:57:57,678 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46761 connecting to ZooKeeper ensemble=127.0.0.1:60762 2024-12-07T21:57:57,678 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T21:57:57,680 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T21:57:57,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:467610x0, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-07T21:57:57,705 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46761-0x1000256da950001 connected 2024-12-07T21:57:57,705 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46761-0x1000256da950001, quorum=127.0.0.1:60762, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T21:57:57,713 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-07T21:57:57,725 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/bucket.cache 2024-12-07T21:57:57,730 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-07T21:57:57,731 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-07T21:57:57,732 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-07T21:57:57,734 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-07T21:57:57,734 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-07T21:57:57,740 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-07T21:57:57,744 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-07T21:57:57,745 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/bucket.persistence is complete. 2024-12-07T21:57:57,746 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-07T21:57:57,749 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46761-0x1000256da950001, quorum=127.0.0.1:60762, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-07T21:57:57,754 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46761-0x1000256da950001, quorum=127.0.0.1:60762, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-07T21:57:57,754 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46761 2024-12-07T21:57:57,755 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46761 2024-12-07T21:57:57,755 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46761 2024-12-07T21:57:57,756 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46761 2024-12-07T21:57:57,756 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46761 2024-12-07T21:57:57,779 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/dbf4292a68c8:0 server-side Connection retries=45 2024-12-07T21:57:57,780 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T21:57:57,780 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-07T21:57:57,780 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-07T21:57:57,780 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T21:57:57,780 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-07T21:57:57,781 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-07T21:57:57,781 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-07T21:57:57,782 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41383 2024-12-07T21:57:57,784 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41383 connecting to ZooKeeper ensemble=127.0.0.1:60762 2024-12-07T21:57:57,784 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T21:57:57,785 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T21:57:57,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:413830x0, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-07T21:57:57,809 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41383-0x1000256da950002, quorum=127.0.0.1:60762, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T21:57:57,809 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41383-0x1000256da950002 connected 2024-12-07T21:57:57,810 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-07T21:57:57,812 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/bucket.cache 2024-12-07T21:57:57,812 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-07T21:57:57,812 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-07T21:57:57,813 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-07T21:57:57,813 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-07T21:57:57,813 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-07T21:57:57,813 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-07T21:57:57,814 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-07T21:57:57,814 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/bucket.persistence is complete. 2024-12-07T21:57:57,814 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-07T21:57:57,816 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41383-0x1000256da950002, quorum=127.0.0.1:60762, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-07T21:57:57,818 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41383-0x1000256da950002, quorum=127.0.0.1:60762, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-07T21:57:57,820 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41383 2024-12-07T21:57:57,821 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41383 2024-12-07T21:57:57,823 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41383 2024-12-07T21:57:57,827 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41383 2024-12-07T21:57:57,828 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41383 2024-12-07T21:57:57,842 DEBUG [M:0;dbf4292a68c8:38375 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;dbf4292a68c8:38375 2024-12-07T21:57:57,844 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/dbf4292a68c8,38375,1733608676689 2024-12-07T21:57:57,859 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46761-0x1000256da950001, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T21:57:57,859 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38375-0x1000256da950000, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T21:57:57,859 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41383-0x1000256da950002, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T21:57:57,861 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38375-0x1000256da950000, quorum=127.0.0.1:60762, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/dbf4292a68c8,38375,1733608676689 2024-12-07T21:57:57,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46761-0x1000256da950001, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-07T21:57:57,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38375-0x1000256da950000, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T21:57:57,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41383-0x1000256da950002, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-07T21:57:57,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46761-0x1000256da950001, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T21:57:57,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41383-0x1000256da950002, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T21:57:57,897 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38375-0x1000256da950000, quorum=127.0.0.1:60762, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-07T21:57:57,898 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/dbf4292a68c8,38375,1733608676689 from backup master directory 2024-12-07T21:57:57,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41383-0x1000256da950002, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T21:57:57,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46761-0x1000256da950001, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T21:57:57,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38375-0x1000256da950000, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/dbf4292a68c8,38375,1733608676689 2024-12-07T21:57:57,909 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38375-0x1000256da950000, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T21:57:57,910 WARN [master/dbf4292a68c8:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-07T21:57:57,910 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=dbf4292a68c8,38375,1733608676689 2024-12-07T21:57:57,913 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-07T21:57:57,915 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-07T21:57:57,922 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/hbase.id] with ID: 8cf967a7-3e14-4f14-a32d-8b3e8b06f207 2024-12-07T21:57:57,923 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/.tmp/hbase.id 2024-12-07T21:57:57,928 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/hbase.id] 2024-12-07T21:57:57,963 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T21:57:57,969 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-07T21:57:57,987 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-12-07T21:57:58,008 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41383-0x1000256da950002, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T21:57:58,008 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38375-0x1000256da950000, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T21:57:58,008 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46761-0x1000256da950001, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T21:57:58,039 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-07T21:57:58,043 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-07T21:57:58,092 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T21:57:58,098 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T21:57:58,149 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/MasterData/data/master/store 2024-12-07T21:57:58,172 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-07T21:57:58,176 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T21:57:58,177 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-07T21:57:58,178 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T21:57:58,178 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T21:57:58,180 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-07T21:57:58,181 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T21:57:58,181 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T21:57:58,182 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733608678177Disabling compacts and flushes for region at 1733608678177Disabling writes for close at 1733608678180 (+3 ms)Writing region close event to WAL at 1733608678181 (+1 ms)Closed at 1733608678181 2024-12-07T21:57:58,182 WARN [master/dbf4292a68c8:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/MasterData/data/master/store/.initializing 2024-12-07T21:57:58,183 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/MasterData/WALs/dbf4292a68c8,38375,1733608676689 2024-12-07T21:57:58,194 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T21:57:58,205 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=dbf4292a68c8%2C38375%2C1733608676689, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/MasterData/WALs/dbf4292a68c8,38375,1733608676689, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/MasterData/oldWALs, maxLogs=10 2024-12-07T21:57:58,236 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/MasterData/WALs/dbf4292a68c8,38375,1733608676689/dbf4292a68c8%2C38375%2C1733608676689.1733608678208 2024-12-07T21:57:58,236 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-07T21:57:58,237 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-07T21:57:58,237 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T21:57:58,241 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T21:57:58,242 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T21:57:58,276 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-07T21:57:58,310 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-07T21:57:58,314 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T21:57:58,316 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T21:57:58,317 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-07T21:57:58,321 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-07T21:57:58,321 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T21:57:58,321 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T21:57:58,322 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-07T21:57:58,326 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-07T21:57:58,326 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T21:57:58,326 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T21:57:58,327 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-07T21:57:58,331 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-07T21:57:58,331 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T21:57:58,332 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T21:57:58,333 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T21:57:58,336 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-07T21:57:58,336 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-07T21:57:58,339 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T21:57:58,339 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T21:57:58,342 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-07T21:57:58,344 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T21:57:58,350 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T21:57:58,351 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64283531, jitterRate=-0.04210074245929718}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-07T21:57:58,360 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733608678254Initializing all the Stores at 1733608678254Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733608678255 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733608678255Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733608678256 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733608678256Cleaning up temporary data from old regions at 1733608678339 (+83 ms)Region opened successfully at 1733608678359 (+20 ms) 2024-12-07T21:57:58,361 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-07T21:57:58,393 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dbbf432, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=dbf4292a68c8/172.17.0.2:0 2024-12-07T21:57:58,429 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-07T21:57:58,444 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-07T21:57:58,444 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-07T21:57:58,447 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T21:57:58,448 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-07T21:57:58,455 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-12-07T21:57:58,455 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-07T21:57:58,484 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-07T21:57:58,493 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38375-0x1000256da950000, quorum=127.0.0.1:60762, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-07T21:57:58,549 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-07T21:57:58,552 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-07T21:57:58,553 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38375-0x1000256da950000, quorum=127.0.0.1:60762, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-07T21:57:58,561 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-07T21:57:58,564 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-07T21:57:58,569 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38375-0x1000256da950000, quorum=127.0.0.1:60762, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-07T21:57:58,578 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-07T21:57:58,581 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38375-0x1000256da950000, quorum=127.0.0.1:60762, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-07T21:57:58,591 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-07T21:57:58,612 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38375-0x1000256da950000, quorum=127.0.0.1:60762, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-07T21:57:58,620 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-07T21:57:58,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41383-0x1000256da950002, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-07T21:57:58,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46761-0x1000256da950001, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-07T21:57:58,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38375-0x1000256da950000, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-07T21:57:58,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41383-0x1000256da950002, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T21:57:58,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46761-0x1000256da950001, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T21:57:58,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38375-0x1000256da950000, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T21:57:58,641 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=dbf4292a68c8,38375,1733608676689, sessionid=0x1000256da950000, setting cluster-up flag (Was=false) 2024-12-07T21:57:58,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46761-0x1000256da950001, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T21:57:58,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38375-0x1000256da950000, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T21:57:58,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41383-0x1000256da950002, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T21:57:58,699 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-07T21:57:58,703 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=dbf4292a68c8,38375,1733608676689 2024-12-07T21:57:58,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46761-0x1000256da950001, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T21:57:58,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38375-0x1000256da950000, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T21:57:58,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41383-0x1000256da950002, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T21:57:58,757 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-07T21:57:58,760 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=dbf4292a68c8,38375,1733608676689 2024-12-07T21:57:58,766 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-07T21:57:58,829 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T21:57:58,832 INFO [RS:0;dbf4292a68c8:46761 {}] regionserver.HRegionServer(746): ClusterId : 8cf967a7-3e14-4f14-a32d-8b3e8b06f207 2024-12-07T21:57:58,832 INFO [RS:1;dbf4292a68c8:41383 {}] regionserver.HRegionServer(746): ClusterId : 8cf967a7-3e14-4f14-a32d-8b3e8b06f207 2024-12-07T21:57:58,834 DEBUG [RS:1;dbf4292a68c8:41383 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-07T21:57:58,834 DEBUG [RS:0;dbf4292a68c8:46761 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-07T21:57:58,839 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-07T21:57:58,847 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-07T21:57:58,852 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: dbf4292a68c8,38375,1733608676689 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-07T21:57:58,855 DEBUG [RS:0;dbf4292a68c8:46761 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-07T21:57:58,855 DEBUG [RS:1;dbf4292a68c8:41383 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-07T21:57:58,856 DEBUG [RS:0;dbf4292a68c8:46761 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-07T21:57:58,856 DEBUG [RS:1;dbf4292a68c8:41383 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-07T21:57:58,867 DEBUG [RS:0;dbf4292a68c8:46761 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-07T21:57:58,867 DEBUG [RS:1;dbf4292a68c8:41383 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-07T21:57:58,868 DEBUG [RS:0;dbf4292a68c8:46761 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3acc9603, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=dbf4292a68c8/172.17.0.2:0 2024-12-07T21:57:58,868 DEBUG [RS:1;dbf4292a68c8:41383 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12498da8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=dbf4292a68c8/172.17.0.2:0 2024-12-07T21:57:58,869 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/dbf4292a68c8:0, corePoolSize=5, maxPoolSize=5 2024-12-07T21:57:58,869 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/dbf4292a68c8:0, corePoolSize=5, maxPoolSize=5 2024-12-07T21:57:58,869 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/dbf4292a68c8:0, corePoolSize=5, maxPoolSize=5 2024-12-07T21:57:58,869 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/dbf4292a68c8:0, corePoolSize=5, maxPoolSize=5 2024-12-07T21:57:58,869 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/dbf4292a68c8:0, corePoolSize=10, maxPoolSize=10 2024-12-07T21:57:58,870 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:57:58,870 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/dbf4292a68c8:0, corePoolSize=2, maxPoolSize=2 2024-12-07T21:57:58,870 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:57:58,875 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-07T21:57:58,875 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-07T21:57:58,884 DEBUG [RS:0;dbf4292a68c8:46761 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;dbf4292a68c8:46761 2024-12-07T21:57:58,884 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T21:57:58,884 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-07T21:57:58,886 DEBUG [RS:1;dbf4292a68c8:41383 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;dbf4292a68c8:41383 2024-12-07T21:57:58,886 INFO [RS:0;dbf4292a68c8:46761 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-07T21:57:58,886 INFO [RS:1;dbf4292a68c8:41383 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-07T21:57:58,887 INFO [RS:0;dbf4292a68c8:46761 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-07T21:57:58,887 INFO [RS:1;dbf4292a68c8:41383 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-07T21:57:58,887 DEBUG [RS:0;dbf4292a68c8:46761 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-07T21:57:58,887 DEBUG [RS:1;dbf4292a68c8:41383 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-07T21:57:58,887 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733608708887 2024-12-07T21:57:58,889 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-07T21:57:58,890 INFO [RS:0;dbf4292a68c8:46761 {}] regionserver.HRegionServer(2659): reportForDuty to master=dbf4292a68c8,38375,1733608676689 with port=46761, startcode=1733608677634 2024-12-07T21:57:58,890 INFO [RS:1;dbf4292a68c8:41383 {}] regionserver.HRegionServer(2659): reportForDuty to master=dbf4292a68c8,38375,1733608676689 with port=41383, startcode=1733608677778 2024-12-07T21:57:58,891 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-07T21:57:58,894 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-07T21:57:58,895 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-07T21:57:58,895 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-07T21:57:58,895 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-07T21:57:58,896 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-07T21:57:58,902 DEBUG [RS:1;dbf4292a68c8:41383 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-07T21:57:58,902 DEBUG [RS:0;dbf4292a68c8:46761 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-07T21:57:58,903 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-07T21:57:58,905 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-07T21:57:58,905 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-07T21:57:58,908 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-07T21:57:58,908 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-07T21:57:58,908 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-07T21:57:58,909 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442 2024-12-07T21:57:58,910 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/dbf4292a68c8:0:becomeActiveMaster-HFileCleaner.large.0-1733608678909,5,FailOnTimeoutGroup] 2024-12-07T21:57:58,919 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/dbf4292a68c8:0:becomeActiveMaster-HFileCleaner.small.0-1733608678910,5,FailOnTimeoutGroup] 2024-12-07T21:57:58,919 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-07T21:57:58,920 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-07T21:57:58,921 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-07T21:57:58,921 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-07T21:57:58,925 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T21:57:58,927 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-07T21:57:58,931 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-07T21:57:58,932 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T21:57:58,932 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T21:57:58,932 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-07T21:57:58,936 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-07T21:57:58,936 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T21:57:58,936 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T21:57:58,936 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-07T21:57:58,940 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-07T21:57:58,940 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T21:57:58,941 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T21:57:58,941 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-07T21:57:58,945 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-07T21:57:58,945 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T21:57:58,945 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T21:57:58,945 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-07T21:57:58,946 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740 2024-12-07T21:57:58,946 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740 2024-12-07T21:57:58,947 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-07T21:57:58,947 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-07T21:57:58,950 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-07T21:57:58,951 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-07T21:57:58,963 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T21:57:58,964 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74763019, jitterRate=0.11405579745769501}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-07T21:57:58,968 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733608678926Initializing all the Stores at 1733608678926Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733608678926Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733608678927 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733608678927Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733608678927Cleaning up temporary data from old regions at 1733608678947 (+20 ms)Region opened successfully at 1733608678968 (+21 ms) 2024-12-07T21:57:58,968 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-07T21:57:58,968 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-07T21:57:58,968 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-07T21:57:58,969 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-07T21:57:58,969 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-07T21:57:58,971 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-07T21:57:58,971 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733608678968Disabling compacts and flushes for region at 1733608678968Disabling writes for close at 1733608678969 (+1 ms)Writing region close event to WAL at 1733608678970 (+1 ms)Closed at 1733608678971 (+1 ms) 2024-12-07T21:57:58,972 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39129, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-07T21:57:58,972 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48101, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-07T21:57:58,974 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-07T21:57:58,975 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-07T21:57:58,979 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38375 {}] master.ServerManager(363): Checking decommissioned status of RegionServer dbf4292a68c8,41383,1733608677778 2024-12-07T21:57:58,982 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-07T21:57:58,982 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38375 {}] master.ServerManager(517): Registering regionserver=dbf4292a68c8,41383,1733608677778 2024-12-07T21:57:58,989 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T21:57:58,991 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-07T21:57:58,995 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38375 {}] master.ServerManager(363): Checking decommissioned status of RegionServer dbf4292a68c8,46761,1733608677634 2024-12-07T21:57:58,995 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38375 {}] master.ServerManager(517): Registering regionserver=dbf4292a68c8,46761,1733608677634 2024-12-07T21:57:58,999 DEBUG [RS:1;dbf4292a68c8:41383 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442 2024-12-07T21:57:58,999 DEBUG [RS:0;dbf4292a68c8:46761 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442 2024-12-07T21:57:58,999 DEBUG [RS:1;dbf4292a68c8:41383 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-07T21:57:58,999 DEBUG [RS:0;dbf4292a68c8:46761 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-07T21:57:58,999 DEBUG [RS:1;dbf4292a68c8:41383 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-07T21:57:58,999 DEBUG [RS:0;dbf4292a68c8:46761 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-07T21:57:59,055 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38375-0x1000256da950000, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-07T21:57:59,056 DEBUG [RS:0;dbf4292a68c8:46761 {}] zookeeper.ZKUtil(111): regionserver:46761-0x1000256da950001, quorum=127.0.0.1:60762, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/dbf4292a68c8,46761,1733608677634 2024-12-07T21:57:59,056 DEBUG [RS:1;dbf4292a68c8:41383 {}] zookeeper.ZKUtil(111): regionserver:41383-0x1000256da950002, quorum=127.0.0.1:60762, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/dbf4292a68c8,41383,1733608677778 2024-12-07T21:57:59,056 WARN [RS:0;dbf4292a68c8:46761 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-07T21:57:59,056 WARN [RS:1;dbf4292a68c8:41383 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-07T21:57:59,056 INFO [RS:0;dbf4292a68c8:46761 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T21:57:59,056 INFO [RS:1;dbf4292a68c8:41383 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T21:57:59,056 DEBUG [RS:1;dbf4292a68c8:41383 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/WALs/dbf4292a68c8,41383,1733608677778 2024-12-07T21:57:59,056 DEBUG [RS:0;dbf4292a68c8:46761 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/WALs/dbf4292a68c8,46761,1733608677634 2024-12-07T21:57:59,060 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [dbf4292a68c8,41383,1733608677778] 2024-12-07T21:57:59,060 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [dbf4292a68c8,46761,1733608677634] 2024-12-07T21:57:59,081 INFO [RS:1;dbf4292a68c8:41383 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-07T21:57:59,081 INFO [RS:0;dbf4292a68c8:46761 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-07T21:57:59,098 INFO [RS:0;dbf4292a68c8:46761 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-07T21:57:59,098 INFO [RS:1;dbf4292a68c8:41383 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-07T21:57:59,103 INFO [RS:1;dbf4292a68c8:41383 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-07T21:57:59,103 INFO [RS:0;dbf4292a68c8:46761 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-07T21:57:59,103 INFO [RS:1;dbf4292a68c8:41383 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T21:57:59,103 INFO [RS:0;dbf4292a68c8:46761 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T21:57:59,104 INFO [RS:1;dbf4292a68c8:41383 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-07T21:57:59,107 INFO [RS:0;dbf4292a68c8:46761 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-07T21:57:59,109 INFO [RS:1;dbf4292a68c8:41383 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-07T21:57:59,109 INFO [RS:0;dbf4292a68c8:46761 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-07T21:57:59,111 INFO [RS:1;dbf4292a68c8:41383 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-07T21:57:59,111 INFO [RS:0;dbf4292a68c8:46761 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-07T21:57:59,112 DEBUG [RS:0;dbf4292a68c8:46761 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:57:59,112 DEBUG [RS:1;dbf4292a68c8:41383 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:57:59,112 DEBUG [RS:0;dbf4292a68c8:46761 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:57:59,112 DEBUG [RS:1;dbf4292a68c8:41383 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:57:59,112 DEBUG [RS:0;dbf4292a68c8:46761 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:57:59,112 DEBUG [RS:1;dbf4292a68c8:41383 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:57:59,112 DEBUG [RS:0;dbf4292a68c8:46761 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:57:59,112 DEBUG [RS:1;dbf4292a68c8:41383 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:57:59,113 DEBUG [RS:0;dbf4292a68c8:46761 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:57:59,113 DEBUG [RS:1;dbf4292a68c8:41383 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:57:59,113 DEBUG [RS:0;dbf4292a68c8:46761 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/dbf4292a68c8:0, corePoolSize=2, maxPoolSize=2 2024-12-07T21:57:59,113 DEBUG [RS:1;dbf4292a68c8:41383 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/dbf4292a68c8:0, corePoolSize=2, maxPoolSize=2 2024-12-07T21:57:59,113 DEBUG [RS:0;dbf4292a68c8:46761 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:57:59,113 DEBUG [RS:1;dbf4292a68c8:41383 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:57:59,113 DEBUG [RS:1;dbf4292a68c8:41383 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:57:59,113 DEBUG [RS:0;dbf4292a68c8:46761 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:57:59,113 DEBUG [RS:1;dbf4292a68c8:41383 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:57:59,113 DEBUG [RS:0;dbf4292a68c8:46761 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:57:59,114 DEBUG [RS:1;dbf4292a68c8:41383 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:57:59,114 DEBUG [RS:0;dbf4292a68c8:46761 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:57:59,114 DEBUG [RS:1;dbf4292a68c8:41383 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:57:59,114 DEBUG [RS:0;dbf4292a68c8:46761 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:57:59,114 DEBUG [RS:1;dbf4292a68c8:41383 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:57:59,114 DEBUG [RS:0;dbf4292a68c8:46761 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:57:59,114 DEBUG [RS:1;dbf4292a68c8:41383 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/dbf4292a68c8:0, corePoolSize=3, maxPoolSize=3 2024-12-07T21:57:59,114 DEBUG [RS:0;dbf4292a68c8:46761 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/dbf4292a68c8:0, corePoolSize=3, maxPoolSize=3 2024-12-07T21:57:59,114 DEBUG [RS:1;dbf4292a68c8:41383 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/dbf4292a68c8:0, corePoolSize=3, maxPoolSize=3 2024-12-07T21:57:59,114 DEBUG [RS:0;dbf4292a68c8:46761 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/dbf4292a68c8:0, corePoolSize=3, maxPoolSize=3 2024-12-07T21:57:59,115 INFO [RS:1;dbf4292a68c8:41383 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T21:57:59,116 INFO [RS:0;dbf4292a68c8:46761 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T21:57:59,116 INFO [RS:1;dbf4292a68c8:41383 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T21:57:59,116 INFO [RS:0;dbf4292a68c8:46761 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T21:57:59,116 INFO [RS:1;dbf4292a68c8:41383 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T21:57:59,116 INFO [RS:0;dbf4292a68c8:46761 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T21:57:59,116 INFO [RS:1;dbf4292a68c8:41383 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-07T21:57:59,116 INFO [RS:0;dbf4292a68c8:46761 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-07T21:57:59,116 INFO [RS:1;dbf4292a68c8:41383 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-07T21:57:59,116 INFO [RS:0;dbf4292a68c8:46761 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-07T21:57:59,116 INFO [RS:1;dbf4292a68c8:41383 {}] hbase.ChoreService(168): Chore ScheduledChore name=dbf4292a68c8,41383,1733608677778-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-07T21:57:59,116 INFO [RS:0;dbf4292a68c8:46761 {}] hbase.ChoreService(168): Chore ScheduledChore name=dbf4292a68c8,46761,1733608677634-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-07T21:57:59,134 INFO [RS:1;dbf4292a68c8:41383 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-07T21:57:59,136 INFO [RS:0;dbf4292a68c8:46761 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-07T21:57:59,136 INFO [RS:1;dbf4292a68c8:41383 {}] hbase.ChoreService(168): Chore ScheduledChore name=dbf4292a68c8,41383,1733608677778-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T21:57:59,136 INFO [RS:0;dbf4292a68c8:46761 {}] hbase.ChoreService(168): Chore ScheduledChore name=dbf4292a68c8,46761,1733608677634-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T21:57:59,137 INFO [RS:0;dbf4292a68c8:46761 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T21:57:59,137 INFO [RS:1;dbf4292a68c8:41383 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T21:57:59,137 INFO [RS:0;dbf4292a68c8:46761 {}] regionserver.Replication(171): dbf4292a68c8,46761,1733608677634 started 2024-12-07T21:57:59,137 INFO [RS:1;dbf4292a68c8:41383 {}] regionserver.Replication(171): dbf4292a68c8,41383,1733608677778 started 2024-12-07T21:57:59,142 WARN [dbf4292a68c8:38375 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-07T21:57:59,162 INFO [RS:1;dbf4292a68c8:41383 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T21:57:59,162 INFO [RS:0;dbf4292a68c8:46761 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T21:57:59,162 INFO [RS:1;dbf4292a68c8:41383 {}] regionserver.HRegionServer(1482): Serving as dbf4292a68c8,41383,1733608677778, RpcServer on dbf4292a68c8/172.17.0.2:41383, sessionid=0x1000256da950002 2024-12-07T21:57:59,162 INFO [RS:0;dbf4292a68c8:46761 {}] regionserver.HRegionServer(1482): Serving as dbf4292a68c8,46761,1733608677634, RpcServer on dbf4292a68c8/172.17.0.2:46761, sessionid=0x1000256da950001 2024-12-07T21:57:59,164 DEBUG [RS:1;dbf4292a68c8:41383 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-07T21:57:59,164 DEBUG [RS:0;dbf4292a68c8:46761 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-07T21:57:59,164 DEBUG [RS:1;dbf4292a68c8:41383 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager dbf4292a68c8,41383,1733608677778 2024-12-07T21:57:59,164 DEBUG [RS:0;dbf4292a68c8:46761 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager dbf4292a68c8,46761,1733608677634 2024-12-07T21:57:59,164 DEBUG [RS:1;dbf4292a68c8:41383 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dbf4292a68c8,41383,1733608677778' 2024-12-07T21:57:59,164 DEBUG [RS:0;dbf4292a68c8:46761 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dbf4292a68c8,46761,1733608677634' 2024-12-07T21:57:59,164 DEBUG [RS:1;dbf4292a68c8:41383 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-07T21:57:59,164 DEBUG [RS:0;dbf4292a68c8:46761 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-07T21:57:59,165 DEBUG [RS:0;dbf4292a68c8:46761 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-07T21:57:59,165 DEBUG [RS:1;dbf4292a68c8:41383 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-07T21:57:59,166 DEBUG [RS:0;dbf4292a68c8:46761 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-07T21:57:59,166 DEBUG [RS:1;dbf4292a68c8:41383 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-07T21:57:59,166 DEBUG [RS:0;dbf4292a68c8:46761 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-07T21:57:59,166 DEBUG [RS:1;dbf4292a68c8:41383 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-07T21:57:59,166 DEBUG [RS:1;dbf4292a68c8:41383 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager dbf4292a68c8,41383,1733608677778 2024-12-07T21:57:59,166 DEBUG [RS:0;dbf4292a68c8:46761 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager dbf4292a68c8,46761,1733608677634 2024-12-07T21:57:59,166 DEBUG [RS:1;dbf4292a68c8:41383 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dbf4292a68c8,41383,1733608677778' 2024-12-07T21:57:59,166 DEBUG [RS:0;dbf4292a68c8:46761 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dbf4292a68c8,46761,1733608677634' 2024-12-07T21:57:59,166 DEBUG [RS:1;dbf4292a68c8:41383 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-07T21:57:59,166 DEBUG [RS:0;dbf4292a68c8:46761 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-07T21:57:59,167 DEBUG [RS:1;dbf4292a68c8:41383 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-07T21:57:59,167 DEBUG [RS:0;dbf4292a68c8:46761 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-07T21:57:59,168 DEBUG [RS:1;dbf4292a68c8:41383 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-07T21:57:59,168 DEBUG [RS:0;dbf4292a68c8:46761 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-07T21:57:59,168 INFO [RS:0;dbf4292a68c8:46761 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-07T21:57:59,168 INFO [RS:1;dbf4292a68c8:41383 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-07T21:57:59,168 INFO [RS:0;dbf4292a68c8:46761 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-07T21:57:59,168 INFO [RS:1;dbf4292a68c8:41383 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-07T21:57:59,277 INFO [RS:1;dbf4292a68c8:41383 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T21:57:59,277 INFO [RS:0;dbf4292a68c8:46761 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T21:57:59,279 INFO [RS:0;dbf4292a68c8:46761 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=dbf4292a68c8%2C46761%2C1733608677634, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/WALs/dbf4292a68c8,46761,1733608677634, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/oldWALs, maxLogs=55 2024-12-07T21:57:59,279 INFO [RS:1;dbf4292a68c8:41383 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=dbf4292a68c8%2C41383%2C1733608677778, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/WALs/dbf4292a68c8,41383,1733608677778, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/oldWALs, maxLogs=55 2024-12-07T21:57:59,299 INFO [RS:1;dbf4292a68c8:41383 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/WALs/dbf4292a68c8,41383,1733608677778/dbf4292a68c8%2C41383%2C1733608677778.1733608679282 2024-12-07T21:57:59,299 INFO [RS:0;dbf4292a68c8:46761 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/WALs/dbf4292a68c8,46761,1733608677634/dbf4292a68c8%2C46761%2C1733608677634.1733608679282 2024-12-07T21:57:59,300 DEBUG [RS:1;dbf4292a68c8:41383 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-07T21:57:59,300 DEBUG [RS:0;dbf4292a68c8:46761 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-07T21:57:59,397 DEBUG [dbf4292a68c8:38375 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-07T21:57:59,410 DEBUG [dbf4292a68c8:38375 {}] balancer.BalancerClusterState(204): Hosts are {dbf4292a68c8=0} racks are {/default-rack=0} 2024-12-07T21:57:59,413 DEBUG [dbf4292a68c8:38375 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-07T21:57:59,413 DEBUG [dbf4292a68c8:38375 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-07T21:57:59,413 DEBUG [dbf4292a68c8:38375 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-07T21:57:59,413 DEBUG [dbf4292a68c8:38375 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-07T21:57:59,413 INFO [dbf4292a68c8:38375 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-07T21:57:59,413 INFO [dbf4292a68c8:38375 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-07T21:57:59,414 DEBUG [dbf4292a68c8:38375 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-07T21:57:59,421 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=dbf4292a68c8,46761,1733608677634 2024-12-07T21:57:59,425 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as dbf4292a68c8,46761,1733608677634, state=OPENING 2024-12-07T21:57:59,461 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-07T21:57:59,470 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46761-0x1000256da950001, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T21:57:59,470 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38375-0x1000256da950000, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T21:57:59,470 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41383-0x1000256da950002, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T21:57:59,471 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T21:57:59,472 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T21:57:59,472 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T21:57:59,474 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T21:57:59,477 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=dbf4292a68c8,46761,1733608677634}] 2024-12-07T21:57:59,650 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-07T21:57:59,653 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43675, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-07T21:57:59,660 INFO [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-07T21:57:59,660 INFO [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T21:57:59,661 INFO [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-07T21:57:59,663 INFO [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=dbf4292a68c8%2C46761%2C1733608677634.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/WALs/dbf4292a68c8,46761,1733608677634, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/oldWALs, maxLogs=55 2024-12-07T21:57:59,680 INFO [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/WALs/dbf4292a68c8,46761,1733608677634/dbf4292a68c8%2C46761%2C1733608677634.meta.1733608679665.meta 2024-12-07T21:57:59,680 DEBUG [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-07T21:57:59,681 DEBUG [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-07T21:57:59,682 DEBUG [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-07T21:57:59,684 DEBUG [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-07T21:57:59,688 INFO [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-07T21:57:59,691 DEBUG [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-07T21:57:59,692 DEBUG [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T21:57:59,692 DEBUG [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-07T21:57:59,692 DEBUG [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-07T21:57:59,694 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-07T21:57:59,695 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-07T21:57:59,695 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T21:57:59,695 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T21:57:59,696 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-07T21:57:59,697 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-07T21:57:59,697 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T21:57:59,697 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T21:57:59,697 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-07T21:57:59,698 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-07T21:57:59,698 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T21:57:59,698 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T21:57:59,699 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-07T21:57:59,699 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-07T21:57:59,700 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T21:57:59,700 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T21:57:59,700 DEBUG [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-07T21:57:59,701 DEBUG [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740 2024-12-07T21:57:59,702 DEBUG [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740 2024-12-07T21:57:59,703 DEBUG [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-07T21:57:59,703 DEBUG [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-07T21:57:59,703 DEBUG [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-07T21:57:59,704 DEBUG [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-07T21:57:59,705 INFO [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58932891, jitterRate=-0.1218314915895462}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-07T21:57:59,705 DEBUG [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-07T21:57:59,708 DEBUG [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733608679693Writing region info on filesystem at 1733608679693Initializing all the Stores at 1733608679693Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733608679693Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733608679694 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733608679694Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733608679694Cleaning up temporary data from old regions at 1733608679703 (+9 ms)Running coprocessor post-open hooks at 1733608679705 (+2 ms)Region opened successfully at 1733608679707 (+2 ms) 2024-12-07T21:57:59,714 INFO [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733608679642 2024-12-07T21:57:59,723 DEBUG [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-07T21:57:59,724 INFO [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-07T21:57:59,725 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=dbf4292a68c8,46761,1733608677634 2024-12-07T21:57:59,726 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as dbf4292a68c8,46761,1733608677634, state=OPEN 2024-12-07T21:57:59,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38375-0x1000256da950000, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T21:57:59,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41383-0x1000256da950002, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T21:57:59,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46761-0x1000256da950001, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T21:57:59,745 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T21:57:59,745 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T21:57:59,745 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T21:57:59,746 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=dbf4292a68c8,46761,1733608677634 2024-12-07T21:57:59,749 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-07T21:57:59,749 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=dbf4292a68c8,46761,1733608677634 in 269 msec 2024-12-07T21:57:59,753 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-07T21:57:59,753 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 769 msec 2024-12-07T21:57:59,754 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-07T21:57:59,754 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-07T21:57:59,771 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T21:57:59,773 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=dbf4292a68c8,46761,1733608677634, seqNum=-1] 2024-12-07T21:57:59,790 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T21:57:59,792 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42967, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T21:57:59,810 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0140 sec 2024-12-07T21:57:59,810 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733608679810, completionTime=-1 2024-12-07T21:57:59,812 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-07T21:57:59,812 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T21:57:59,841 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T21:57:59,841 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733608739841 2024-12-07T21:57:59,841 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733608799841 2024-12-07T21:57:59,841 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 28 msec 2024-12-07T21:57:59,843 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-07T21:57:59,849 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dbf4292a68c8,38375,1733608676689-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T21:57:59,849 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dbf4292a68c8,38375,1733608676689-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T21:57:59,850 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dbf4292a68c8,38375,1733608676689-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T21:57:59,851 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-dbf4292a68c8:38375, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T21:57:59,851 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-07T21:57:59,851 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-07T21:57:59,857 DEBUG [master/dbf4292a68c8:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-07T21:57:59,874 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.963sec 2024-12-07T21:57:59,875 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-07T21:57:59,877 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-07T21:57:59,877 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-07T21:57:59,878 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-07T21:57:59,878 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-07T21:57:59,879 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dbf4292a68c8,38375,1733608676689-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-07T21:57:59,879 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dbf4292a68c8,38375,1733608676689-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-07T21:57:59,880 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-07T21:57:59,881 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-07T21:57:59,882 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dbf4292a68c8,38375,1733608676689-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T21:57:59,959 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@319823c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T21:57:59,960 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-07T21:57:59,960 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-07T21:57:59,964 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request dbf4292a68c8,38375,-1 for getting cluster id 2024-12-07T21:57:59,966 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T21:57:59,975 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8cf967a7-3e14-4f14-a32d-8b3e8b06f207' 2024-12-07T21:57:59,977 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T21:57:59,977 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8cf967a7-3e14-4f14-a32d-8b3e8b06f207" 2024-12-07T21:57:59,977 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47a4677, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T21:57:59,978 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [dbf4292a68c8,38375,-1] 2024-12-07T21:57:59,979 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T21:57:59,981 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T21:57:59,982 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58648, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T21:57:59,984 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27097cb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T21:57:59,984 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T21:57:59,990 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=dbf4292a68c8,46761,1733608677634, seqNum=-1] 2024-12-07T21:57:59,990 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T21:57:59,993 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55750, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T21:58:00,011 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=dbf4292a68c8,38375,1733608676689 2024-12-07T21:58:00,014 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-07T21:58:00,018 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is dbf4292a68c8,38375,1733608676689 2024-12-07T21:58:00,020 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5cc99567 2024-12-07T21:58:00,021 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-07T21:58:00,023 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58652, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-07T21:58:00,028 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1733608680011', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-07T21:58:00,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1733608680011 2024-12-07T21:58:00,036 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733608680011 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-07T21:58:00,038 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1733608680011" procId is: 4 2024-12-07T21:58:00,038 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T21:58:00,040 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733608680011 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-07T21:58:00,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T21:58:00,051 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733608680011-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fd5b361aff2e6cdaf4dbf54f71c0a4da, NAME => 'testBlockEvictionOnRegionMove1733608680011,,1733608680025.fd5b361aff2e6cdaf4dbf54f71c0a4da.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1733608680011', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442 2024-12-07T21:58:00,058 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733608680011-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733608680011,,1733608680025.fd5b361aff2e6cdaf4dbf54f71c0a4da.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T21:58:00,059 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733608680011-pool-0 {}] regionserver.HRegion(1722): Closing fd5b361aff2e6cdaf4dbf54f71c0a4da, disabling compactions & flushes 2024-12-07T21:58:00,059 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733608680011-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733608680011,,1733608680025.fd5b361aff2e6cdaf4dbf54f71c0a4da. 2024-12-07T21:58:00,059 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733608680011-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733608680011,,1733608680025.fd5b361aff2e6cdaf4dbf54f71c0a4da. 2024-12-07T21:58:00,059 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733608680011-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733608680011,,1733608680025.fd5b361aff2e6cdaf4dbf54f71c0a4da. after waiting 0 ms 2024-12-07T21:58:00,059 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733608680011-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733608680011,,1733608680025.fd5b361aff2e6cdaf4dbf54f71c0a4da. 2024-12-07T21:58:00,059 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733608680011-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733608680011,,1733608680025.fd5b361aff2e6cdaf4dbf54f71c0a4da. 2024-12-07T21:58:00,059 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733608680011-pool-0 {}] regionserver.HRegion(1676): Region close journal for fd5b361aff2e6cdaf4dbf54f71c0a4da: Waiting for close lock at 1733608680059Disabling compacts and flushes for region at 1733608680059Disabling writes for close at 1733608680059Writing region close event to WAL at 1733608680059Closed at 1733608680059 2024-12-07T21:58:00,060 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733608680011 execute state=CREATE_TABLE_ADD_TO_META 2024-12-07T21:58:00,064 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1733608680011,,1733608680025.fd5b361aff2e6cdaf4dbf54f71c0a4da.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733608680060"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733608680060"}]},"ts":"1733608680060"} 2024-12-07T21:58:00,068 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-07T21:58:00,069 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733608680011 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-07T21:58:00,071 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1733608680011","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733608680069"}]},"ts":"1733608680069"} 2024-12-07T21:58:00,074 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1733608680011, state=ENABLING in hbase:meta 2024-12-07T21:58:00,075 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {dbf4292a68c8=0} racks are {/default-rack=0} 2024-12-07T21:58:00,075 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-07T21:58:00,075 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-07T21:58:00,075 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-07T21:58:00,075 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-07T21:58:00,075 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-07T21:58:00,075 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-07T21:58:00,075 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-07T21:58:00,076 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733608680011, region=fd5b361aff2e6cdaf4dbf54f71c0a4da, ASSIGN}] 2024-12-07T21:58:00,078 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733608680011, region=fd5b361aff2e6cdaf4dbf54f71c0a4da, ASSIGN 2024-12-07T21:58:00,078 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733608680011, region=fd5b361aff2e6cdaf4dbf54f71c0a4da, ASSIGN; state=OFFLINE, location=dbf4292a68c8,41383,1733608677778; forceNewPlan=false, retain=false 2024-12-07T21:58:00,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T21:58:00,231 INFO [dbf4292a68c8:38375 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-07T21:58:00,232 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=fd5b361aff2e6cdaf4dbf54f71c0a4da, regionState=OPENING, regionLocation=dbf4292a68c8,41383,1733608677778 2024-12-07T21:58:00,237 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733608680011, region=fd5b361aff2e6cdaf4dbf54f71c0a4da, ASSIGN because future has completed 2024-12-07T21:58:00,238 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure fd5b361aff2e6cdaf4dbf54f71c0a4da, server=dbf4292a68c8,41383,1733608677778}] 2024-12-07T21:58:00,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T21:58:00,391 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-07T21:58:00,393 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42825, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-07T21:58:00,395 INFO [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1733608680011,,1733608680025.fd5b361aff2e6cdaf4dbf54f71c0a4da. 2024-12-07T21:58:00,396 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => fd5b361aff2e6cdaf4dbf54f71c0a4da, NAME => 'testBlockEvictionOnRegionMove1733608680011,,1733608680025.fd5b361aff2e6cdaf4dbf54f71c0a4da.', STARTKEY => '', ENDKEY => ''} 2024-12-07T21:58:00,396 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1733608680011 fd5b361aff2e6cdaf4dbf54f71c0a4da 2024-12-07T21:58:00,396 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733608680011,,1733608680025.fd5b361aff2e6cdaf4dbf54f71c0a4da.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T21:58:00,397 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for fd5b361aff2e6cdaf4dbf54f71c0a4da 2024-12-07T21:58:00,397 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for fd5b361aff2e6cdaf4dbf54f71c0a4da 2024-12-07T21:58:00,398 INFO [StoreOpener-fd5b361aff2e6cdaf4dbf54f71c0a4da-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region fd5b361aff2e6cdaf4dbf54f71c0a4da 2024-12-07T21:58:00,403 INFO [StoreOpener-fd5b361aff2e6cdaf4dbf54f71c0a4da-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fd5b361aff2e6cdaf4dbf54f71c0a4da columnFamilyName family 2024-12-07T21:58:00,403 DEBUG [StoreOpener-fd5b361aff2e6cdaf4dbf54f71c0a4da-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T21:58:00,404 INFO [StoreOpener-fd5b361aff2e6cdaf4dbf54f71c0a4da-1 {}] regionserver.HStore(327): Store=fd5b361aff2e6cdaf4dbf54f71c0a4da/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T21:58:00,404 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for fd5b361aff2e6cdaf4dbf54f71c0a4da 2024-12-07T21:58:00,406 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/default/testBlockEvictionOnRegionMove1733608680011/fd5b361aff2e6cdaf4dbf54f71c0a4da 2024-12-07T21:58:00,406 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/default/testBlockEvictionOnRegionMove1733608680011/fd5b361aff2e6cdaf4dbf54f71c0a4da 2024-12-07T21:58:00,406 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for fd5b361aff2e6cdaf4dbf54f71c0a4da 2024-12-07T21:58:00,406 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for fd5b361aff2e6cdaf4dbf54f71c0a4da 2024-12-07T21:58:00,407 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for fd5b361aff2e6cdaf4dbf54f71c0a4da 2024-12-07T21:58:00,414 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/default/testBlockEvictionOnRegionMove1733608680011/fd5b361aff2e6cdaf4dbf54f71c0a4da/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T21:58:00,415 INFO [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened fd5b361aff2e6cdaf4dbf54f71c0a4da; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59943174, jitterRate=-0.10677710175514221}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T21:58:00,416 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fd5b361aff2e6cdaf4dbf54f71c0a4da 2024-12-07T21:58:00,417 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for fd5b361aff2e6cdaf4dbf54f71c0a4da: Running coprocessor pre-open hook at 1733608680397Writing region info on filesystem at 1733608680397Initializing all the Stores at 1733608680397Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733608680398 (+1 ms)Cleaning up temporary data from old regions at 1733608680406 (+8 ms)Running coprocessor post-open hooks at 1733608680416 (+10 ms)Region opened successfully at 1733608680417 (+1 ms) 2024-12-07T21:58:00,418 INFO [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1733608680011,,1733608680025.fd5b361aff2e6cdaf4dbf54f71c0a4da., pid=6, masterSystemTime=1733608680391 2024-12-07T21:58:00,421 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1733608680011,,1733608680025.fd5b361aff2e6cdaf4dbf54f71c0a4da. 2024-12-07T21:58:00,421 INFO [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1733608680011,,1733608680025.fd5b361aff2e6cdaf4dbf54f71c0a4da. 2024-12-07T21:58:00,421 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=fd5b361aff2e6cdaf4dbf54f71c0a4da, regionState=OPEN, openSeqNum=2, regionLocation=dbf4292a68c8,41383,1733608677778 2024-12-07T21:58:00,424 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure fd5b361aff2e6cdaf4dbf54f71c0a4da, server=dbf4292a68c8,41383,1733608677778 because future has completed 2024-12-07T21:58:00,427 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-07T21:58:00,427 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure fd5b361aff2e6cdaf4dbf54f71c0a4da, server=dbf4292a68c8,41383,1733608677778 in 188 msec 2024-12-07T21:58:00,429 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-07T21:58:00,429 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733608680011, region=fd5b361aff2e6cdaf4dbf54f71c0a4da, ASSIGN in 351 msec 2024-12-07T21:58:00,430 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733608680011 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-07T21:58:00,430 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1733608680011","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733608680430"}]},"ts":"1733608680430"} 2024-12-07T21:58:00,433 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1733608680011, state=ENABLED in hbase:meta 2024-12-07T21:58:00,434 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733608680011 execute state=CREATE_TABLE_POST_OPERATION 2024-12-07T21:58:00,435 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1733608680011 in 403 msec 2024-12-07T21:58:00,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T21:58:00,682 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1733608680011 get assigned. Timeout = 60000ms 2024-12-07T21:58:00,682 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1733608680011 completed 2024-12-07T21:58:00,687 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-07T21:58:00,692 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1733608680011 assigned to meta. Checking AM states. 2024-12-07T21:58:00,692 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-07T21:58:00,693 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1733608680011 assigned. 2024-12-07T21:58:00,699 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1733608680011', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1733608680011,,1733608680025.fd5b361aff2e6cdaf4dbf54f71c0a4da., hostname=dbf4292a68c8,41383,1733608677778, seqNum=2] 2024-12-07T21:58:00,700 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T21:58:00,702 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51922, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T21:58:00,712 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing fd5b361aff2e6cdaf4dbf54f71c0a4da 1/1 column families, dataSize=78 B heapSize=480 B 2024-12-07T21:58:00,733 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a9cf9a320c57445e8f86cfe8da07f378 2024-12-07T21:58:00,770 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/default/testBlockEvictionOnRegionMove1733608680011/fd5b361aff2e6cdaf4dbf54f71c0a4da/.tmp/family/a9cf9a320c57445e8f86cfe8da07f378 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-12-07T21:58:00,775 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/default/testBlockEvictionOnRegionMove1733608680011/fd5b361aff2e6cdaf4dbf54f71c0a4da/.tmp/family/a9cf9a320c57445e8f86cfe8da07f378 2024-12-07T21:58:00,811 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/default/testBlockEvictionOnRegionMove1733608680011/fd5b361aff2e6cdaf4dbf54f71c0a4da/.tmp/family/a9cf9a320c57445e8f86cfe8da07f378 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/default/testBlockEvictionOnRegionMove1733608680011/fd5b361aff2e6cdaf4dbf54f71c0a4da/family/a9cf9a320c57445e8f86cfe8da07f378 2024-12-07T21:58:00,812 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/default/testBlockEvictionOnRegionMove1733608680011/fd5b361aff2e6cdaf4dbf54f71c0a4da/family/a9cf9a320c57445e8f86cfe8da07f378, delay=912 ms 2024-12-07T21:58:00,814 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/default/testBlockEvictionOnRegionMove1733608680011/fd5b361aff2e6cdaf4dbf54f71c0a4da/family/a9cf9a320c57445e8f86cfe8da07f378, entries=2, sequenceid=6, filesize=4.9 K 2024-12-07T21:58:00,820 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for fd5b361aff2e6cdaf4dbf54f71c0a4da in 107ms, sequenceid=6, compaction requested=false 2024-12-07T21:58:00,821 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1733608680011' 2024-12-07T21:58:00,823 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for fd5b361aff2e6cdaf4dbf54f71c0a4da: 2024-12-07T21:58:01,730 DEBUG [hfile-prefetch-1733608680812 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a9cf9a320c57445e8f86cfe8da07f378_0. 2024-12-07T21:58:01,739 DEBUG [hfile-prefetch-1733608680812 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a9cf9a320c57445e8f86cfe8da07f378_117. 2024-12-07T21:58:01,740 DEBUG [hfile-prefetch-1733608680812 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/default/testBlockEvictionOnRegionMove1733608680011/fd5b361aff2e6cdaf4dbf54f71c0a4da/family/a9cf9a320c57445e8f86cfe8da07f378, with total blocks 2, and data blocks 1 2024-12-07T21:58:01,740 DEBUG [hfile-prefetch-1733608680812 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-07T21:58:01,741 DEBUG [hfile-prefetch-1733608680812 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/default/testBlockEvictionOnRegionMove1733608680011/fd5b361aff2e6cdaf4dbf54f71c0a4da/family/a9cf9a320c57445e8f86cfe8da07f378, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-12-07T21:58:01,741 DEBUG [hfile-prefetch-1733608680812 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/default/testBlockEvictionOnRegionMove1733608680011/fd5b361aff2e6cdaf4dbf54f71c0a4da/family/a9cf9a320c57445e8f86cfe8da07f378. Will sleep 100ms and try the verification again. 2024-12-07T21:58:01,817 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-07T21:58:01,817 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-07T21:58:01,818 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-07T21:58:01,818 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-12-07T21:58:01,832 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=fd5b361aff2e6cdaf4dbf54f71c0a4da, source=dbf4292a68c8,41383,1733608677778, destination=dbf4292a68c8,46761,1733608677634, warming up region on dbf4292a68c8,46761,1733608677634 2024-12-07T21:58:01,834 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=fd5b361aff2e6cdaf4dbf54f71c0a4da, source=dbf4292a68c8,41383,1733608677778, destination=dbf4292a68c8,46761,1733608677634, running balancer 2024-12-07T21:58:01,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733608680011, region=fd5b361aff2e6cdaf4dbf54f71c0a4da, REOPEN/MOVE 2024-12-07T21:58:01,835 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733608680011, region=fd5b361aff2e6cdaf4dbf54f71c0a4da, REOPEN/MOVE 2024-12-07T21:58:01,836 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1733608680011,,1733608680025.fd5b361aff2e6cdaf4dbf54f71c0a4da. 2024-12-07T21:58:01,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] regionserver.HRegion(7855): Warmup {ENCODED => fd5b361aff2e6cdaf4dbf54f71c0a4da, NAME => 'testBlockEvictionOnRegionMove1733608680011,,1733608680025.fd5b361aff2e6cdaf4dbf54f71c0a4da.', STARTKEY => '', ENDKEY => ''} 2024-12-07T21:58:01,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733608680011,,1733608680025.fd5b361aff2e6cdaf4dbf54f71c0a4da.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T21:58:01,836 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-07T21:58:01,837 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-07T21:58:01,837 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-07T21:58:01,837 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-07T21:58:01,837 INFO [StoreOpener-fd5b361aff2e6cdaf4dbf54f71c0a4da-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region fd5b361aff2e6cdaf4dbf54f71c0a4da 2024-12-07T21:58:01,837 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=fd5b361aff2e6cdaf4dbf54f71c0a4da, regionState=CLOSING, regionLocation=dbf4292a68c8,41383,1733608677778 2024-12-07T21:58:01,838 INFO [StoreOpener-fd5b361aff2e6cdaf4dbf54f71c0a4da-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fd5b361aff2e6cdaf4dbf54f71c0a4da columnFamilyName family 2024-12-07T21:58:01,838 DEBUG [StoreOpener-fd5b361aff2e6cdaf4dbf54f71c0a4da-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T21:58:01,839 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-12-07T21:58:01,839 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1733608681817 2024-12-07T21:58:01,839 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-07T21:58:01,840 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-07T21:58:01,840 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733608681837 2024-12-07T21:58:01,840 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-07T21:58:01,840 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733608680011, region=fd5b361aff2e6cdaf4dbf54f71c0a4da, REOPEN/MOVE because future has completed 2024-12-07T21:58:01,841 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-12-07T21:58:01,841 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure fd5b361aff2e6cdaf4dbf54f71c0a4da, server=dbf4292a68c8,41383,1733608677778}] 2024-12-07T21:58:01,841 DEBUG [hfile-prefetch-1733608680812 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/default/testBlockEvictionOnRegionMove1733608680011/fd5b361aff2e6cdaf4dbf54f71c0a4da/family/a9cf9a320c57445e8f86cfe8da07f378, with total blocks 2, and data blocks 1 2024-12-07T21:58:01,841 DEBUG [hfile-prefetch-1733608680812 {}] bucket.BucketCache(2285): iterating over 1 entries in the backing map 2024-12-07T21:58:01,842 DEBUG [hfile-prefetch-1733608680812 {}] bucket.BucketCache(2293): found block for file a9cf9a320c57445e8f86cfe8da07f378 in the backing map. Acquiring read lock for offset 0 2024-12-07T21:58:01,842 DEBUG [hfile-prefetch-1733608680812 {}] bucket.BucketCache(2305): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/default/testBlockEvictionOnRegionMove1733608680011/fd5b361aff2e6cdaf4dbf54f71c0a4da/family/a9cf9a320c57445e8f86cfe8da07f378 has now been fully cached. 2024-12-07T21:58:01,842 DEBUG [StoreOpener-fd5b361aff2e6cdaf4dbf54f71c0a4da-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/default/testBlockEvictionOnRegionMove1733608680011/fd5b361aff2e6cdaf4dbf54f71c0a4da/family/a9cf9a320c57445e8f86cfe8da07f378 2024-12-07T21:58:01,842 DEBUG [hfile-prefetch-1733608680812 {}] hfile.PrefetchExecutor(121): Prefetch completed for a9cf9a320c57445e8f86cfe8da07f378 2024-12-07T21:58:01,842 INFO [StoreOpener-fd5b361aff2e6cdaf4dbf54f71c0a4da-1 {}] regionserver.HStore(327): Store=fd5b361aff2e6cdaf4dbf54f71c0a4da/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T21:58:01,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] regionserver.HRegion(1722): Closing fd5b361aff2e6cdaf4dbf54f71c0a4da, disabling compactions & flushes 2024-12-07T21:58:01,843 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733608680011,,1733608680025.fd5b361aff2e6cdaf4dbf54f71c0a4da. 2024-12-07T21:58:01,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733608680011,,1733608680025.fd5b361aff2e6cdaf4dbf54f71c0a4da. 2024-12-07T21:58:01,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733608680011,,1733608680025.fd5b361aff2e6cdaf4dbf54f71c0a4da. after waiting 0 ms 2024-12-07T21:58:01,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733608680011,,1733608680025.fd5b361aff2e6cdaf4dbf54f71c0a4da. 2024-12-07T21:58:01,844 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733608680011,,1733608680025.fd5b361aff2e6cdaf4dbf54f71c0a4da. 2024-12-07T21:58:01,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46761 {}] regionserver.HRegion(1676): Region close journal for fd5b361aff2e6cdaf4dbf54f71c0a4da: Waiting for close lock at 1733608681843Disabling compacts and flushes for region at 1733608681843Disabling writes for close at 1733608681843Writing region close event to WAL at 1733608681844 (+1 ms)Closed at 1733608681844 2024-12-07T21:58:01,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38375 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-07T21:58:02,002 INFO [RS_CLOSE_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close fd5b361aff2e6cdaf4dbf54f71c0a4da 2024-12-07T21:58:02,003 DEBUG [RS_CLOSE_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-12-07T21:58:02,005 DEBUG [RS_CLOSE_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing fd5b361aff2e6cdaf4dbf54f71c0a4da, disabling compactions & flushes 2024-12-07T21:58:02,005 INFO [RS_CLOSE_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733608680011,,1733608680025.fd5b361aff2e6cdaf4dbf54f71c0a4da. 2024-12-07T21:58:02,005 DEBUG [RS_CLOSE_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733608680011,,1733608680025.fd5b361aff2e6cdaf4dbf54f71c0a4da. 2024-12-07T21:58:02,005 DEBUG [RS_CLOSE_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733608680011,,1733608680025.fd5b361aff2e6cdaf4dbf54f71c0a4da. after waiting 0 ms 2024-12-07T21:58:02,005 DEBUG [RS_CLOSE_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733608680011,,1733608680025.fd5b361aff2e6cdaf4dbf54f71c0a4da. 2024-12-07T21:58:02,008 DEBUG [StoreFileCloser-fd5b361aff2e6cdaf4dbf54f71c0a4da-family-1 {}] bucket.BucketCache(768): Removing file a9cf9a320c57445e8f86cfe8da07f378 for region fd5b361aff2e6cdaf4dbf54f71c0a4da 2024-12-07T21:58:02,009 DEBUG [StoreFileCloser-fd5b361aff2e6cdaf4dbf54f71c0a4da-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file a9cf9a320c57445e8f86cfe8da07f378, starting offset: 0, end offset: 9223372036854775807 2024-12-07T21:58:02,011 DEBUG [StoreFileCloser-fd5b361aff2e6cdaf4dbf54f71c0a4da-family-1 {}] bucket.BucketCache(856): removed key a9cf9a320c57445e8f86cfe8da07f378_0 from back map with offset lock 2094080 in the evict process 2024-12-07T21:58:02,015 DEBUG [RS_CLOSE_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/default/testBlockEvictionOnRegionMove1733608680011/fd5b361aff2e6cdaf4dbf54f71c0a4da/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-07T21:58:02,017 INFO [RS_CLOSE_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733608680011,,1733608680025.fd5b361aff2e6cdaf4dbf54f71c0a4da. 2024-12-07T21:58:02,017 DEBUG [RS_CLOSE_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for fd5b361aff2e6cdaf4dbf54f71c0a4da: Waiting for close lock at 1733608682004Running coprocessor pre-close hooks at 1733608682004Disabling compacts and flushes for region at 1733608682004Disabling writes for close at 1733608682005 (+1 ms)Writing region close event to WAL at 1733608682011 (+6 ms)Running coprocessor post-close hooks at 1733608682016 (+5 ms)Closed at 1733608682016 2024-12-07T21:58:02,017 INFO [RS_CLOSE_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding fd5b361aff2e6cdaf4dbf54f71c0a4da move to dbf4292a68c8,46761,1733608677634 record at close sequenceid=6 2024-12-07T21:58:02,021 INFO [RS_CLOSE_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed fd5b361aff2e6cdaf4dbf54f71c0a4da 2024-12-07T21:58:02,021 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=fd5b361aff2e6cdaf4dbf54f71c0a4da, regionState=CLOSED 2024-12-07T21:58:02,024 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure fd5b361aff2e6cdaf4dbf54f71c0a4da, server=dbf4292a68c8,41383,1733608677778 because future has completed 2024-12-07T21:58:02,027 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-07T21:58:02,027 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure fd5b361aff2e6cdaf4dbf54f71c0a4da, server=dbf4292a68c8,41383,1733608677778 in 184 msec 2024-12-07T21:58:02,027 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733608680011, region=fd5b361aff2e6cdaf4dbf54f71c0a4da, REOPEN/MOVE; state=CLOSED, location=dbf4292a68c8,46761,1733608677634; forceNewPlan=false, retain=false 2024-12-07T21:58:02,039 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-07T21:58:02,039 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-07T21:58:02,039 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-07T21:58:02,039 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-07T21:58:02,040 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-07T21:58:02,040 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-07T21:58:02,040 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-07T21:58:02,041 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-07T21:58:02,043 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-07T21:58:02,043 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733608682039 2024-12-07T21:58:02,043 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-07T21:58:02,044 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-07T21:58:02,044 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733608682040 2024-12-07T21:58:02,044 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-07T21:58:02,178 INFO [dbf4292a68c8:38375 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-07T21:58:02,179 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=fd5b361aff2e6cdaf4dbf54f71c0a4da, regionState=OPENING, regionLocation=dbf4292a68c8,46761,1733608677634 2024-12-07T21:58:02,184 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733608680011, region=fd5b361aff2e6cdaf4dbf54f71c0a4da, REOPEN/MOVE because future has completed 2024-12-07T21:58:02,185 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure fd5b361aff2e6cdaf4dbf54f71c0a4da, server=dbf4292a68c8,46761,1733608677634}] 2024-12-07T21:58:02,344 INFO [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1733608680011,,1733608680025.fd5b361aff2e6cdaf4dbf54f71c0a4da. 2024-12-07T21:58:02,345 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => fd5b361aff2e6cdaf4dbf54f71c0a4da, NAME => 'testBlockEvictionOnRegionMove1733608680011,,1733608680025.fd5b361aff2e6cdaf4dbf54f71c0a4da.', STARTKEY => '', ENDKEY => ''} 2024-12-07T21:58:02,345 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1733608680011 fd5b361aff2e6cdaf4dbf54f71c0a4da 2024-12-07T21:58:02,345 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733608680011,,1733608680025.fd5b361aff2e6cdaf4dbf54f71c0a4da.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T21:58:02,345 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for fd5b361aff2e6cdaf4dbf54f71c0a4da 2024-12-07T21:58:02,345 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for fd5b361aff2e6cdaf4dbf54f71c0a4da 2024-12-07T21:58:02,346 INFO [StoreOpener-fd5b361aff2e6cdaf4dbf54f71c0a4da-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region fd5b361aff2e6cdaf4dbf54f71c0a4da 2024-12-07T21:58:02,348 INFO [StoreOpener-fd5b361aff2e6cdaf4dbf54f71c0a4da-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fd5b361aff2e6cdaf4dbf54f71c0a4da columnFamilyName family 2024-12-07T21:58:02,348 DEBUG [StoreOpener-fd5b361aff2e6cdaf4dbf54f71c0a4da-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T21:58:02,350 DEBUG [StoreFileOpener-fd5b361aff2e6cdaf4dbf54f71c0a4da-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/default/testBlockEvictionOnRegionMove1733608680011/fd5b361aff2e6cdaf4dbf54f71c0a4da/family/a9cf9a320c57445e8f86cfe8da07f378, delay=936 ms 2024-12-07T21:58:02,351 DEBUG [StoreOpener-fd5b361aff2e6cdaf4dbf54f71c0a4da-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/default/testBlockEvictionOnRegionMove1733608680011/fd5b361aff2e6cdaf4dbf54f71c0a4da/family/a9cf9a320c57445e8f86cfe8da07f378 2024-12-07T21:58:02,351 INFO [StoreOpener-fd5b361aff2e6cdaf4dbf54f71c0a4da-1 {}] regionserver.HStore(327): Store=fd5b361aff2e6cdaf4dbf54f71c0a4da/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T21:58:02,352 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for fd5b361aff2e6cdaf4dbf54f71c0a4da 2024-12-07T21:58:02,353 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/default/testBlockEvictionOnRegionMove1733608680011/fd5b361aff2e6cdaf4dbf54f71c0a4da 2024-12-07T21:58:02,353 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/default/testBlockEvictionOnRegionMove1733608680011/fd5b361aff2e6cdaf4dbf54f71c0a4da 2024-12-07T21:58:02,353 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for fd5b361aff2e6cdaf4dbf54f71c0a4da 2024-12-07T21:58:02,353 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for fd5b361aff2e6cdaf4dbf54f71c0a4da 2024-12-07T21:58:02,359 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for fd5b361aff2e6cdaf4dbf54f71c0a4da 2024-12-07T21:58:02,360 INFO [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened fd5b361aff2e6cdaf4dbf54f71c0a4da; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72081626, jitterRate=0.07409992814064026}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T21:58:02,360 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fd5b361aff2e6cdaf4dbf54f71c0a4da 2024-12-07T21:58:02,361 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for fd5b361aff2e6cdaf4dbf54f71c0a4da: Running coprocessor pre-open hook at 1733608682346Writing region info on filesystem at 1733608682346Initializing all the Stores at 1733608682346Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733608682346Cleaning up temporary data from old regions at 1733608682353 (+7 ms)Running coprocessor post-open hooks at 1733608682360 (+7 ms)Region opened successfully at 1733608682361 (+1 ms) 2024-12-07T21:58:02,362 INFO [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1733608680011,,1733608680025.fd5b361aff2e6cdaf4dbf54f71c0a4da., pid=9, masterSystemTime=1733608682338 2024-12-07T21:58:02,365 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1733608680011,,1733608680025.fd5b361aff2e6cdaf4dbf54f71c0a4da. 2024-12-07T21:58:02,365 INFO [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1733608680011,,1733608680025.fd5b361aff2e6cdaf4dbf54f71c0a4da. 2024-12-07T21:58:02,366 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=fd5b361aff2e6cdaf4dbf54f71c0a4da, regionState=OPEN, openSeqNum=10, regionLocation=dbf4292a68c8,46761,1733608677634 2024-12-07T21:58:02,368 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure fd5b361aff2e6cdaf4dbf54f71c0a4da, server=dbf4292a68c8,46761,1733608677634 because future has completed 2024-12-07T21:58:02,371 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-07T21:58:02,372 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure fd5b361aff2e6cdaf4dbf54f71c0a4da, server=dbf4292a68c8,46761,1733608677634 in 185 msec 2024-12-07T21:58:02,373 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733608680011, region=fd5b361aff2e6cdaf4dbf54f71c0a4da, REOPEN/MOVE in 537 msec 2024-12-07T21:58:02,388 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-07T21:58:02,389 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-07T21:58:02,389 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T21:58:02,392 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T21:58:02,393 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T21:58:02,393 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T21:58:02,393 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-07T21:58:02,393 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2143720667, stopped=false 2024-12-07T21:58:02,394 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=dbf4292a68c8,38375,1733608676689 2024-12-07T21:58:02,423 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41383-0x1000256da950002, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-07T21:58:02,423 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38375-0x1000256da950000, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-07T21:58:02,423 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46761-0x1000256da950001, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-07T21:58:02,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41383-0x1000256da950002, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T21:58:02,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38375-0x1000256da950000, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T21:58:02,424 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T21:58:02,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46761-0x1000256da950001, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T21:58:02,425 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41383-0x1000256da950002, quorum=127.0.0.1:60762, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T21:58:02,425 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46761-0x1000256da950001, quorum=127.0.0.1:60762, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T21:58:02,426 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-07T21:58:02,426 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T21:58:02,426 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38375-0x1000256da950000, quorum=127.0.0.1:60762, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T21:58:02,426 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T21:58:02,427 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'dbf4292a68c8,46761,1733608677634' ***** 2024-12-07T21:58:02,427 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-07T21:58:02,427 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'dbf4292a68c8,41383,1733608677778' ***** 2024-12-07T21:58:02,428 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-07T21:58:02,428 INFO [RS:0;dbf4292a68c8:46761 {}] bucket.BucketCache(1738): Disabling cache 2024-12-07T21:58:02,428 INFO [RS:1;dbf4292a68c8:41383 {}] bucket.BucketCache(1738): Disabling cache 2024-12-07T21:58:02,429 INFO [RS:0;dbf4292a68c8:46761 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/bucket.persistence 2024-12-07T21:58:02,429 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-07T21:58:02,429 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-07T21:58:02,429 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-07T21:58:02,429 INFO [RS:1;dbf4292a68c8:41383 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/bucket.persistence 2024-12-07T21:58:02,429 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-07T21:58:02,430 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-07T21:58:02,430 INFO [RS:1;dbf4292a68c8:41383 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-07T21:58:02,430 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-07T21:58:02,430 INFO [RS:0;dbf4292a68c8:46761 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-07T21:58:02,430 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-07T21:58:02,430 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-07T21:58:02,440 DEBUG [RS:1;dbf4292a68c8:41383 {}] bucket.BucketCache(1487): Thread RS:1;dbf4292a68c8:41383 started persisting bucket cache to file 2024-12-07T21:58:02,440 DEBUG [RS:1;dbf4292a68c8:41383 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-07T21:58:02,440 DEBUG [RS:0;dbf4292a68c8:46761 {}] bucket.BucketCache(1487): Thread RS:0;dbf4292a68c8:46761 started persisting bucket cache to file 2024-12-07T21:58:02,440 DEBUG [RS:1;dbf4292a68c8:41383 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-07T21:58:02,440 DEBUG [RS:0;dbf4292a68c8:46761 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-07T21:58:02,440 DEBUG [RS:0;dbf4292a68c8:46761 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-07T21:58:02,443 DEBUG [RS:0;dbf4292a68c8:46761 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-07T21:58:02,443 DEBUG [RS:1;dbf4292a68c8:41383 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-07T21:58:02,443 DEBUG [RS:0;dbf4292a68c8:46761 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733608682440 2024-12-07T21:58:02,443 DEBUG [RS:1;dbf4292a68c8:41383 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733608682440 2024-12-07T21:58:02,443 DEBUG [RS:1;dbf4292a68c8:41383 {}] bucket.BucketCache(1509): Thread RS:1;dbf4292a68c8:41383 finished persisting bucket cache to file, renaming 2024-12-07T21:58:02,443 INFO [RS:1;dbf4292a68c8:41383 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-07T21:58:02,444 INFO [RS:1;dbf4292a68c8:41383 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-07T21:58:02,444 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-07T21:58:02,444 INFO [RS:1;dbf4292a68c8:41383 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-07T21:58:02,444 INFO [RS:1;dbf4292a68c8:41383 {}] regionserver.HRegionServer(959): stopping server dbf4292a68c8,41383,1733608677778 2024-12-07T21:58:02,444 INFO [RS:1;dbf4292a68c8:41383 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-07T21:58:02,444 INFO [RS:1;dbf4292a68c8:41383 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;dbf4292a68c8:41383. 2024-12-07T21:58:02,444 DEBUG [RS:1;dbf4292a68c8:41383 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T21:58:02,444 DEBUG [RS:1;dbf4292a68c8:41383 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T21:58:02,445 INFO [RS:1;dbf4292a68c8:41383 {}] regionserver.HRegionServer(976): stopping server dbf4292a68c8,41383,1733608677778; all regions closed. 2024-12-07T21:58:02,449 DEBUG [RS:1;dbf4292a68c8:41383 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/oldWALs 2024-12-07T21:58:02,450 INFO [RS:1;dbf4292a68c8:41383 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL dbf4292a68c8%2C41383%2C1733608677778:(num 1733608679282) 2024-12-07T21:58:02,450 DEBUG [RS:1;dbf4292a68c8:41383 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T21:58:02,450 INFO [RS:1;dbf4292a68c8:41383 {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T21:58:02,450 INFO [RS:1;dbf4292a68c8:41383 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-07T21:58:02,450 INFO [RS:1;dbf4292a68c8:41383 {}] hbase.ChoreService(370): Chore service for: regionserver/dbf4292a68c8:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-07T21:58:02,451 INFO [regionserver/dbf4292a68c8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-07T21:58:02,451 INFO [RS:1;dbf4292a68c8:41383 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-07T21:58:02,451 INFO [RS:1;dbf4292a68c8:41383 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-07T21:58:02,451 INFO [RS:1;dbf4292a68c8:41383 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-07T21:58:02,451 INFO [RS:1;dbf4292a68c8:41383 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-07T21:58:02,451 INFO [RS:1;dbf4292a68c8:41383 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41383 2024-12-07T21:58:02,456 DEBUG [RS:0;dbf4292a68c8:46761 {}] bucket.BucketCache(1509): Thread RS:0;dbf4292a68c8:46761 finished persisting bucket cache to file, renaming 2024-12-07T21:58:02,456 WARN [RS:0;dbf4292a68c8:46761 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-12-07T21:58:02,456 INFO [RS:0;dbf4292a68c8:46761 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-07T21:58:02,456 INFO [RS:0;dbf4292a68c8:46761 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-07T21:58:02,456 INFO [RS:0;dbf4292a68c8:46761 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-07T21:58:02,456 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-07T21:58:02,456 INFO [RS:0;dbf4292a68c8:46761 {}] regionserver.HRegionServer(3091): Received CLOSE for fd5b361aff2e6cdaf4dbf54f71c0a4da 2024-12-07T21:58:02,457 INFO [RS:0;dbf4292a68c8:46761 {}] regionserver.HRegionServer(959): stopping server dbf4292a68c8,46761,1733608677634 2024-12-07T21:58:02,457 INFO [RS:0;dbf4292a68c8:46761 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-07T21:58:02,457 INFO [RS:0;dbf4292a68c8:46761 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;dbf4292a68c8:46761. 2024-12-07T21:58:02,457 DEBUG [RS:0;dbf4292a68c8:46761 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T21:58:02,457 DEBUG [RS:0;dbf4292a68c8:46761 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T21:58:02,457 INFO [RS:0;dbf4292a68c8:46761 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-07T21:58:02,457 INFO [RS:0;dbf4292a68c8:46761 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-07T21:58:02,457 INFO [RS:0;dbf4292a68c8:46761 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-07T21:58:02,457 INFO [RS_CLOSE_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1733608680011,,1733608680025.fd5b361aff2e6cdaf4dbf54f71c0a4da. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-07T21:58:02,457 INFO [RS:0;dbf4292a68c8:46761 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-07T21:58:02,458 DEBUG [RS_CLOSE_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fd5b361aff2e6cdaf4dbf54f71c0a4da, disabling compactions & flushes 2024-12-07T21:58:02,458 INFO [RS_CLOSE_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733608680011,,1733608680025.fd5b361aff2e6cdaf4dbf54f71c0a4da. 2024-12-07T21:58:02,458 INFO [RS:0;dbf4292a68c8:46761 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-07T21:58:02,458 DEBUG [RS_CLOSE_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733608680011,,1733608680025.fd5b361aff2e6cdaf4dbf54f71c0a4da. 2024-12-07T21:58:02,458 DEBUG [RS:0;dbf4292a68c8:46761 {}] regionserver.HRegionServer(1325): Online Regions={fd5b361aff2e6cdaf4dbf54f71c0a4da=testBlockEvictionOnRegionMove1733608680011,,1733608680025.fd5b361aff2e6cdaf4dbf54f71c0a4da., 1588230740=hbase:meta,,1.1588230740} 2024-12-07T21:58:02,458 INFO [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-07T21:58:02,458 DEBUG [RS_CLOSE_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733608680011,,1733608680025.fd5b361aff2e6cdaf4dbf54f71c0a4da. after waiting 0 ms 2024-12-07T21:58:02,458 DEBUG [RS_CLOSE_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733608680011,,1733608680025.fd5b361aff2e6cdaf4dbf54f71c0a4da. 2024-12-07T21:58:02,458 DEBUG [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-07T21:58:02,458 INFO [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-07T21:58:02,458 DEBUG [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-07T21:58:02,458 DEBUG [RS:0;dbf4292a68c8:46761 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, fd5b361aff2e6cdaf4dbf54f71c0a4da 2024-12-07T21:58:02,458 DEBUG [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-07T21:58:02,458 DEBUG [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-07T21:58:02,458 INFO [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-12-07T21:58:02,458 DEBUG [StoreFileCloser-fd5b361aff2e6cdaf4dbf54f71c0a4da-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/default/testBlockEvictionOnRegionMove1733608680011/fd5b361aff2e6cdaf4dbf54f71c0a4da/family/a9cf9a320c57445e8f86cfe8da07f378 2024-12-07T21:58:02,462 DEBUG [RS_CLOSE_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/default/testBlockEvictionOnRegionMove1733608680011/fd5b361aff2e6cdaf4dbf54f71c0a4da/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-12-07T21:58:02,463 INFO [RS_CLOSE_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733608680011,,1733608680025.fd5b361aff2e6cdaf4dbf54f71c0a4da. 2024-12-07T21:58:02,463 DEBUG [RS_CLOSE_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fd5b361aff2e6cdaf4dbf54f71c0a4da: Waiting for close lock at 1733608682457Running coprocessor pre-close hooks at 1733608682457Disabling compacts and flushes for region at 1733608682458 (+1 ms)Disabling writes for close at 1733608682458Writing region close event to WAL at 1733608682459 (+1 ms)Running coprocessor post-close hooks at 1733608682463 (+4 ms)Closed at 1733608682463 2024-12-07T21:58:02,463 DEBUG [RS_CLOSE_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1733608680011,,1733608680025.fd5b361aff2e6cdaf4dbf54f71c0a4da. 2024-12-07T21:58:02,467 DEBUG [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 0ba7109654964ceab2c0a583cdc6047b 2024-12-07T21:58:02,482 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38375-0x1000256da950000, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-07T21:58:02,482 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41383-0x1000256da950002, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/dbf4292a68c8,41383,1733608677778 2024-12-07T21:58:02,482 INFO [RS:1;dbf4292a68c8:41383 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-07T21:58:02,483 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [dbf4292a68c8,41383,1733608677778] 2024-12-07T21:58:02,486 DEBUG [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740/.tmp/info/0ba7109654964ceab2c0a583cdc6047b is 205, key is testBlockEvictionOnRegionMove1733608680011,,1733608680025.fd5b361aff2e6cdaf4dbf54f71c0a4da./info:regioninfo/1733608682366/Put/seqid=0 2024-12-07T21:58:02,487 INFO [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740/.tmp/info/0ba7109654964ceab2c0a583cdc6047b 2024-12-07T21:58:02,496 DEBUG [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 94182a5eca094c08b4cf92b4eb1708e5 2024-12-07T21:58:02,503 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/dbf4292a68c8,41383,1733608677778 already deleted, retry=false 2024-12-07T21:58:02,503 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; dbf4292a68c8,41383,1733608677778 expired; onlineServers=1 2024-12-07T21:58:02,507 DEBUG [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740/.tmp/ns/94182a5eca094c08b4cf92b4eb1708e5 is 43, key is default/ns:d/1733608679796/Put/seqid=0 2024-12-07T21:58:02,508 INFO [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740/.tmp/ns/94182a5eca094c08b4cf92b4eb1708e5 2024-12-07T21:58:02,514 DEBUG [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 2bebca89dfb946f4b040a5138ad1d2e0 2024-12-07T21:58:02,520 INFO [regionserver/dbf4292a68c8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T21:58:02,520 INFO [regionserver/dbf4292a68c8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T21:58:02,525 DEBUG [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740/.tmp/table/2bebca89dfb946f4b040a5138ad1d2e0 is 78, key is testBlockEvictionOnRegionMove1733608680011/table:state/1733608680430/Put/seqid=0 2024-12-07T21:58:02,526 INFO [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740/.tmp/table/2bebca89dfb946f4b040a5138ad1d2e0 2024-12-07T21:58:02,528 DEBUG [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740/.tmp/info/0ba7109654964ceab2c0a583cdc6047b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740/info/0ba7109654964ceab2c0a583cdc6047b 2024-12-07T21:58:02,529 DEBUG [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740/info/0ba7109654964ceab2c0a583cdc6047b, delay=969 ms 2024-12-07T21:58:02,530 INFO [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740/info/0ba7109654964ceab2c0a583cdc6047b, entries=15, sequenceid=15, filesize=7.6 K 2024-12-07T21:58:02,531 DEBUG [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740/.tmp/ns/94182a5eca094c08b4cf92b4eb1708e5 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740/ns/94182a5eca094c08b4cf92b4eb1708e5 2024-12-07T21:58:02,532 DEBUG [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740/ns/94182a5eca094c08b4cf92b4eb1708e5, delay=994 ms 2024-12-07T21:58:02,533 INFO [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740/ns/94182a5eca094c08b4cf92b4eb1708e5, entries=2, sequenceid=15, filesize=5.0 K 2024-12-07T21:58:02,533 DEBUG [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740/.tmp/table/2bebca89dfb946f4b040a5138ad1d2e0 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740/table/2bebca89dfb946f4b040a5138ad1d2e0 2024-12-07T21:58:02,534 DEBUG [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740/table/2bebca89dfb946f4b040a5138ad1d2e0, delay=943 ms 2024-12-07T21:58:02,535 INFO [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740/table/2bebca89dfb946f4b040a5138ad1d2e0, entries=2, sequenceid=15, filesize=5.3 K 2024-12-07T21:58:02,536 INFO [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 78ms, sequenceid=15, compaction requested=false 2024-12-07T21:58:02,536 DEBUG [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-07T21:58:02,539 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740/info/0ba7109654964ceab2c0a583cdc6047b 2024-12-07T21:58:02,540 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740/ns/94182a5eca094c08b4cf92b4eb1708e5 2024-12-07T21:58:02,540 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740/table/2bebca89dfb946f4b040a5138ad1d2e0 2024-12-07T21:58:02,545 DEBUG [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-12-07T21:58:02,546 DEBUG [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-07T21:58:02,546 INFO [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-07T21:58:02,546 DEBUG [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733608682458Running coprocessor pre-close hooks at 1733608682458Disabling compacts and flushes for region at 1733608682458Disabling writes for close at 1733608682458Obtaining lock to block concurrent updates at 1733608682458Preparing flush snapshotting stores in 1588230740 at 1733608682458Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1733608682459 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733608682459Flushing 1588230740/info: creating writer at 1733608682459Flushing 1588230740/info: appending metadata at 1733608682484 (+25 ms)Flushing 1588230740/info: closing flushed file at 1733608682484Flushing 1588230740/ns: creating writer at 1733608682491 (+7 ms)Flushing 1588230740/ns: appending metadata at 1733608682506 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1733608682506Flushing 1588230740/table: creating writer at 1733608682510 (+4 ms)Flushing 1588230740/table: appending metadata at 1733608682525 (+15 ms)Flushing 1588230740/table: closing flushed file at 1733608682525Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19e63f6f: reopening flushed file at 1733608682528 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f7ea140: reopening flushed file at 1733608682530 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@55944fe5: reopening flushed file at 1733608682533 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 78ms, sequenceid=15, compaction requested=false at 1733608682536 (+3 ms)Writing region close event to WAL at 1733608682541 (+5 ms)Running coprocessor post-close hooks at 1733608682546 (+5 ms)Closed at 1733608682546 2024-12-07T21:58:02,547 DEBUG [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-07T21:58:02,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41383-0x1000256da950002, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T21:58:02,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41383-0x1000256da950002, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T21:58:02,596 INFO [RS:1;dbf4292a68c8:41383 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-07T21:58:02,597 INFO [RS:1;dbf4292a68c8:41383 {}] regionserver.HRegionServer(1031): Exiting; stopping=dbf4292a68c8,41383,1733608677778; zookeeper connection closed. 2024-12-07T21:58:02,597 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1d7dedf3 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1d7dedf3 2024-12-07T21:58:02,658 INFO [RS:0;dbf4292a68c8:46761 {}] regionserver.HRegionServer(976): stopping server dbf4292a68c8,46761,1733608677634; all regions closed. 2024-12-07T21:58:02,662 DEBUG [RS:0;dbf4292a68c8:46761 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/oldWALs 2024-12-07T21:58:02,662 INFO [RS:0;dbf4292a68c8:46761 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL dbf4292a68c8%2C46761%2C1733608677634.meta:.meta(num 1733608679665) 2024-12-07T21:58:02,664 DEBUG [RS:0;dbf4292a68c8:46761 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/oldWALs 2024-12-07T21:58:02,664 INFO [RS:0;dbf4292a68c8:46761 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL dbf4292a68c8%2C46761%2C1733608677634:(num 1733608679282) 2024-12-07T21:58:02,664 DEBUG [RS:0;dbf4292a68c8:46761 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T21:58:02,664 INFO [RS:0;dbf4292a68c8:46761 {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T21:58:02,664 INFO [RS:0;dbf4292a68c8:46761 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-07T21:58:02,665 INFO [RS:0;dbf4292a68c8:46761 {}] hbase.ChoreService(370): Chore service for: regionserver/dbf4292a68c8:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-07T21:58:02,665 INFO [RS:0;dbf4292a68c8:46761 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-07T21:58:02,665 INFO [regionserver/dbf4292a68c8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-07T21:58:02,665 INFO [RS:0;dbf4292a68c8:46761 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46761 2024-12-07T21:58:02,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46761-0x1000256da950001, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/dbf4292a68c8,46761,1733608677634 2024-12-07T21:58:02,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38375-0x1000256da950000, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-07T21:58:02,682 INFO [RS:0;dbf4292a68c8:46761 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-07T21:58:02,690 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [dbf4292a68c8,46761,1733608677634] 2024-12-07T21:58:02,698 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/dbf4292a68c8,46761,1733608677634 already deleted, retry=false 2024-12-07T21:58:02,698 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; dbf4292a68c8,46761,1733608677634 expired; onlineServers=0 2024-12-07T21:58:02,698 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'dbf4292a68c8,38375,1733608676689' ***** 2024-12-07T21:58:02,698 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-07T21:58:02,698 INFO [M:0;dbf4292a68c8:38375 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-07T21:58:02,699 INFO [M:0;dbf4292a68c8:38375 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-07T21:58:02,699 DEBUG [M:0;dbf4292a68c8:38375 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-07T21:58:02,699 DEBUG [M:0;dbf4292a68c8:38375 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-07T21:58:02,699 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-07T21:58:02,699 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster-HFileCleaner.large.0-1733608678909 {}] cleaner.HFileCleaner(306): Exit Thread[master/dbf4292a68c8:0:becomeActiveMaster-HFileCleaner.large.0-1733608678909,5,FailOnTimeoutGroup] 2024-12-07T21:58:02,699 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster-HFileCleaner.small.0-1733608678910 {}] cleaner.HFileCleaner(306): Exit Thread[master/dbf4292a68c8:0:becomeActiveMaster-HFileCleaner.small.0-1733608678910,5,FailOnTimeoutGroup] 2024-12-07T21:58:02,699 INFO [M:0;dbf4292a68c8:38375 {}] hbase.ChoreService(370): Chore service for: master/dbf4292a68c8:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-07T21:58:02,699 INFO [M:0;dbf4292a68c8:38375 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-07T21:58:02,699 DEBUG [M:0;dbf4292a68c8:38375 {}] master.HMaster(1795): Stopping service threads 2024-12-07T21:58:02,699 INFO [M:0;dbf4292a68c8:38375 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-07T21:58:02,700 INFO [M:0;dbf4292a68c8:38375 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T21:58:02,700 INFO [M:0;dbf4292a68c8:38375 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T21:58:02,700 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-07T21:58:02,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38375-0x1000256da950000, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-07T21:58:02,707 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38375-0x1000256da950000, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T21:58:02,707 DEBUG [M:0;dbf4292a68c8:38375 {}] zookeeper.ZKUtil(347): master:38375-0x1000256da950000, quorum=127.0.0.1:60762, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-07T21:58:02,707 WARN [M:0;dbf4292a68c8:38375 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-07T21:58:02,707 INFO [M:0;dbf4292a68c8:38375 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/.lastflushedseqids 2024-12-07T21:58:02,718 INFO [M:0;dbf4292a68c8:38375 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T21:58:02,718 INFO [M:0;dbf4292a68c8:38375 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-07T21:58:02,719 DEBUG [M:0;dbf4292a68c8:38375 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-07T21:58:02,719 INFO [M:0;dbf4292a68c8:38375 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T21:58:02,719 DEBUG [M:0;dbf4292a68c8:38375 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T21:58:02,719 DEBUG [M:0;dbf4292a68c8:38375 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-07T21:58:02,719 DEBUG [M:0;dbf4292a68c8:38375 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T21:58:02,719 INFO [M:0;dbf4292a68c8:38375 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.69 KB heapSize=43.02 KB 2024-12-07T21:58:02,728 DEBUG [M:0;dbf4292a68c8:38375 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 126ba08d296c4f29a14cf05f124ed59d 2024-12-07T21:58:02,741 DEBUG [M:0;dbf4292a68c8:38375 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/126ba08d296c4f29a14cf05f124ed59d is 82, key is hbase:meta,,1/info:regioninfo/1733608679724/Put/seqid=0 2024-12-07T21:58:02,742 INFO [M:0;dbf4292a68c8:38375 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/126ba08d296c4f29a14cf05f124ed59d 2024-12-07T21:58:02,748 DEBUG [M:0;dbf4292a68c8:38375 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 5f8ea923c33a40e8af538ca2eef4c9a5 2024-12-07T21:58:02,761 DEBUG [M:0;dbf4292a68c8:38375 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5f8ea923c33a40e8af538ca2eef4c9a5 is 779, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733608680435/Put/seqid=0 2024-12-07T21:58:02,762 INFO [M:0;dbf4292a68c8:38375 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.07 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5f8ea923c33a40e8af538ca2eef4c9a5 2024-12-07T21:58:02,769 DEBUG [M:0;dbf4292a68c8:38375 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 1f1273dcda1843b190aeda8e83e746a4 2024-12-07T21:58:02,780 DEBUG [M:0;dbf4292a68c8:38375 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1f1273dcda1843b190aeda8e83e746a4 is 69, key is dbf4292a68c8,41383,1733608677778/rs:state/1733608678985/Put/seqid=0 2024-12-07T21:58:02,781 INFO [M:0;dbf4292a68c8:38375 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1f1273dcda1843b190aeda8e83e746a4 2024-12-07T21:58:02,783 DEBUG [M:0;dbf4292a68c8:38375 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/126ba08d296c4f29a14cf05f124ed59d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/126ba08d296c4f29a14cf05f124ed59d 2024-12-07T21:58:02,784 INFO [M:0;dbf4292a68c8:38375 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/126ba08d296c4f29a14cf05f124ed59d, entries=8, sequenceid=84, filesize=5.5 K 2024-12-07T21:58:02,785 DEBUG [M:0;dbf4292a68c8:38375 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5f8ea923c33a40e8af538ca2eef4c9a5 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/5f8ea923c33a40e8af538ca2eef4c9a5 2024-12-07T21:58:02,786 INFO [M:0;dbf4292a68c8:38375 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/5f8ea923c33a40e8af538ca2eef4c9a5, entries=9, sequenceid=84, filesize=6.5 K 2024-12-07T21:58:02,786 DEBUG [M:0;dbf4292a68c8:38375 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1f1273dcda1843b190aeda8e83e746a4 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/1f1273dcda1843b190aeda8e83e746a4 2024-12-07T21:58:02,788 INFO [M:0;dbf4292a68c8:38375 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/1f1273dcda1843b190aeda8e83e746a4, entries=2, sequenceid=84, filesize=5.1 K 2024-12-07T21:58:02,789 INFO [M:0;dbf4292a68c8:38375 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 70ms, sequenceid=84, compaction requested=false 2024-12-07T21:58:02,790 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46761-0x1000256da950001, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T21:58:02,790 INFO [RS:0;dbf4292a68c8:46761 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-07T21:58:02,790 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46761-0x1000256da950001, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T21:58:02,790 INFO [RS:0;dbf4292a68c8:46761 {}] regionserver.HRegionServer(1031): Exiting; stopping=dbf4292a68c8,46761,1733608677634; zookeeper connection closed. 2024-12-07T21:58:02,790 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7d6b0cfd {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7d6b0cfd 2024-12-07T21:58:02,791 INFO [M:0;dbf4292a68c8:38375 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T21:58:02,791 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-12-07T21:58:02,791 DEBUG [M:0;dbf4292a68c8:38375 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733608682719Disabling compacts and flushes for region at 1733608682719Disabling writes for close at 1733608682719Obtaining lock to block concurrent updates at 1733608682719Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733608682719Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35518, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1733608682719Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733608682720 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733608682720Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733608682740 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733608682740Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733608682744 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733608682761 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733608682761Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733608682764 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733608682780 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733608682780Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23f78113: reopening flushed file at 1733608682782 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3feab15c: reopening flushed file at 1733608682784 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f30c1f: reopening flushed file at 1733608682786 (+2 ms)Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 70ms, sequenceid=84, compaction requested=false at 1733608682789 (+3 ms)Writing region close event to WAL at 1733608682791 (+2 ms)Closed at 1733608682791 2024-12-07T21:58:02,792 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-07T21:58:02,793 INFO [M:0;dbf4292a68c8:38375 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-07T21:58:02,793 INFO [M:0;dbf4292a68c8:38375 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38375 2024-12-07T21:58:02,793 INFO [M:0;dbf4292a68c8:38375 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-07T21:58:02,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38375-0x1000256da950000, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T21:58:02,924 INFO [M:0;dbf4292a68c8:38375 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-07T21:58:02,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38375-0x1000256da950000, quorum=127.0.0.1:60762, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T21:58:02,933 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-07T21:58:02,951 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-07T21:58:02,951 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T21:58:02,958 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=51 (was 12) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/dbf4292a68c8:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733608682532 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: regionserver/dbf4292a68c8:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733608682350 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/dbf4292a68c8:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: regionserver/dbf4292a68c8:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733608682529 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733608680812 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=477 (was 427) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=18062 (was 18422) 2024-12-07T21:58:02,964 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=51, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=477, ProcessCount=11, AvailableMemoryMB=18061 2024-12-07T21:58:02,964 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/hadoop-log-dir so I do NOT create it in target/test-data/05a8dc90-5822-7fba-c9a9-13ddf09bc5ac 2024-12-07T21:58:02,964 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/hadoop-tmp-dir so I do NOT create it in target/test-data/05a8dc90-5822-7fba-c9a9-13ddf09bc5ac 2024-12-07T21:58:02,964 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T21:58:02,966 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05a8dc90-5822-7fba-c9a9-13ddf09bc5ac/cluster_18b8dd43-c94c-0999-2be5-30442753271c, deleteOnExit=true 2024-12-07T21:58:02,996 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05a8dc90-5822-7fba-c9a9-13ddf09bc5ac/cluster_18b8dd43-c94c-0999-2be5-30442753271c/zookeeper_0, clientPort=65259, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05a8dc90-5822-7fba-c9a9-13ddf09bc5ac/cluster_18b8dd43-c94c-0999-2be5-30442753271c/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05a8dc90-5822-7fba-c9a9-13ddf09bc5ac/cluster_18b8dd43-c94c-0999-2be5-30442753271c/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-07T21:58:02,997 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=65259 2024-12-07T21:58:03,014 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442 with version=8 2024-12-07T21:58:03,014 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/hbase-staging 2024-12-07T21:58:03,017 INFO [Time-limited test {}] client.ConnectionUtils(128): master/dbf4292a68c8:0 server-side Connection retries=45 2024-12-07T21:58:03,018 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T21:58:03,018 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-07T21:58:03,018 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-07T21:58:03,018 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T21:58:03,018 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-07T21:58:03,018 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-07T21:58:03,019 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-07T21:58:03,019 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42165 2024-12-07T21:58:03,021 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42165 connecting to ZooKeeper ensemble=127.0.0.1:65259 2024-12-07T21:58:03,058 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:421650x0, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-07T21:58:03,059 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42165-0x1000256f48f0000 connected 2024-12-07T21:58:03,159 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T21:58:03,160 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T21:58:03,163 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42165-0x1000256f48f0000, quorum=127.0.0.1:65259, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T21:58:03,163 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442, hbase.cluster.distributed=false 2024-12-07T21:58:03,168 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42165-0x1000256f48f0000, quorum=127.0.0.1:65259, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-07T21:58:03,169 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42165 2024-12-07T21:58:03,169 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42165 2024-12-07T21:58:03,169 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42165 2024-12-07T21:58:03,170 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42165 2024-12-07T21:58:03,170 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42165 2024-12-07T21:58:03,184 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/dbf4292a68c8:0 server-side Connection retries=45 2024-12-07T21:58:03,184 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T21:58:03,184 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-07T21:58:03,184 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-07T21:58:03,184 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T21:58:03,184 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-07T21:58:03,184 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-07T21:58:03,184 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-07T21:58:03,185 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40365 2024-12-07T21:58:03,187 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40365 connecting to ZooKeeper ensemble=127.0.0.1:65259 2024-12-07T21:58:03,187 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T21:58:03,187 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T21:58:03,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:403650x0, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-07T21:58:03,196 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40365-0x1000256f48f0001 connected 2024-12-07T21:58:03,196 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40365-0x1000256f48f0001, quorum=127.0.0.1:65259, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T21:58:03,196 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-07T21:58:03,197 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05a8dc90-5822-7fba-c9a9-13ddf09bc5ac/bucket.cache 2024-12-07T21:58:03,197 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-07T21:58:03,197 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-07T21:58:03,197 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-07T21:58:03,198 INFO [Thread-121 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-07T21:58:03,198 WARN [Thread-121 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-07T21:58:03,198 INFO [Thread-121 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-07T21:58:03,198 INFO [Thread-121 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05a8dc90-5822-7fba-c9a9-13ddf09bc5ac/bucket.persistence is complete. 2024-12-07T21:58:03,198 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05a8dc90-5822-7fba-c9a9-13ddf09bc5ac/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05a8dc90-5822-7fba-c9a9-13ddf09bc5ac/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-07T21:58:03,199 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-07T21:58:03,199 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40365-0x1000256f48f0001, quorum=127.0.0.1:65259, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-07T21:58:03,201 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40365-0x1000256f48f0001, quorum=127.0.0.1:65259, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-07T21:58:03,201 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40365 2024-12-07T21:58:03,202 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40365 2024-12-07T21:58:03,203 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40365 2024-12-07T21:58:03,203 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40365 2024-12-07T21:58:03,203 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40365 2024-12-07T21:58:03,218 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/dbf4292a68c8:0 server-side Connection retries=45 2024-12-07T21:58:03,218 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T21:58:03,218 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-07T21:58:03,218 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-07T21:58:03,218 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T21:58:03,218 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-07T21:58:03,218 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-07T21:58:03,218 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-07T21:58:03,219 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35277 2024-12-07T21:58:03,220 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35277 connecting to ZooKeeper ensemble=127.0.0.1:65259 2024-12-07T21:58:03,220 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T21:58:03,220 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T21:58:03,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:352770x0, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-07T21:58:03,229 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35277-0x1000256f48f0002 connected 2024-12-07T21:58:03,229 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35277-0x1000256f48f0002, quorum=127.0.0.1:65259, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T21:58:03,229 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-07T21:58:03,230 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05a8dc90-5822-7fba-c9a9-13ddf09bc5ac/bucket.cache 2024-12-07T21:58:03,230 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-07T21:58:03,230 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-07T21:58:03,230 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-07T21:58:03,231 INFO [Thread-122 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-07T21:58:03,231 WARN [Thread-122 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-07T21:58:03,231 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05a8dc90-5822-7fba-c9a9-13ddf09bc5ac/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05a8dc90-5822-7fba-c9a9-13ddf09bc5ac/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-07T21:58:03,231 INFO [Thread-122 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-07T21:58:03,231 INFO [Thread-122 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05a8dc90-5822-7fba-c9a9-13ddf09bc5ac/bucket.persistence is complete. 2024-12-07T21:58:03,231 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-07T21:58:03,232 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35277-0x1000256f48f0002, quorum=127.0.0.1:65259, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-07T21:58:03,233 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35277-0x1000256f48f0002, quorum=127.0.0.1:65259, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-07T21:58:03,234 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35277 2024-12-07T21:58:03,234 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35277 2024-12-07T21:58:03,235 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35277 2024-12-07T21:58:03,237 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35277 2024-12-07T21:58:03,237 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35277 2024-12-07T21:58:03,247 DEBUG [M:0;dbf4292a68c8:42165 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;dbf4292a68c8:42165 2024-12-07T21:58:03,248 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/dbf4292a68c8,42165,1733608683017 2024-12-07T21:58:03,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42165-0x1000256f48f0000, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T21:58:03,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35277-0x1000256f48f0002, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T21:58:03,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40365-0x1000256f48f0001, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T21:58:03,259 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42165-0x1000256f48f0000, quorum=127.0.0.1:65259, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/dbf4292a68c8,42165,1733608683017 2024-12-07T21:58:03,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42165-0x1000256f48f0000, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T21:58:03,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35277-0x1000256f48f0002, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-07T21:58:03,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40365-0x1000256f48f0001, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-07T21:58:03,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35277-0x1000256f48f0002, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T21:58:03,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40365-0x1000256f48f0001, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T21:58:03,271 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42165-0x1000256f48f0000, quorum=127.0.0.1:65259, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-07T21:58:03,271 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/dbf4292a68c8,42165,1733608683017 from backup master directory 2024-12-07T21:58:03,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42165-0x1000256f48f0000, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/dbf4292a68c8,42165,1733608683017 2024-12-07T21:58:03,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35277-0x1000256f48f0002, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T21:58:03,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40365-0x1000256f48f0001, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T21:58:03,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42165-0x1000256f48f0000, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T21:58:03,278 WARN [master/dbf4292a68c8:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-07T21:58:03,278 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=dbf4292a68c8,42165,1733608683017 2024-12-07T21:58:03,280 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/hbase.id] with ID: 9823098d-1157-414f-951a-96aad5b923e1 2024-12-07T21:58:03,280 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/.tmp/hbase.id 2024-12-07T21:58:03,282 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/hbase.id] 2024-12-07T21:58:03,311 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T21:58:03,311 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-07T21:58:03,313 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-12-07T21:58:03,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35277-0x1000256f48f0002, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T21:58:03,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40365-0x1000256f48f0001, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T21:58:03,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42165-0x1000256f48f0000, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T21:58:03,331 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-07T21:58:03,332 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-07T21:58:03,332 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T21:58:03,353 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/MasterData/data/master/store 2024-12-07T21:58:03,361 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T21:58:03,362 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-07T21:58:03,362 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T21:58:03,362 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T21:58:03,362 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-07T21:58:03,362 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T21:58:03,362 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T21:58:03,362 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733608683362Disabling compacts and flushes for region at 1733608683362Disabling writes for close at 1733608683362Writing region close event to WAL at 1733608683362Closed at 1733608683362 2024-12-07T21:58:03,363 WARN [master/dbf4292a68c8:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/MasterData/data/master/store/.initializing 2024-12-07T21:58:03,363 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/MasterData/WALs/dbf4292a68c8,42165,1733608683017 2024-12-07T21:58:03,371 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T21:58:03,374 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=dbf4292a68c8%2C42165%2C1733608683017, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/MasterData/WALs/dbf4292a68c8,42165,1733608683017, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/MasterData/oldWALs, maxLogs=10 2024-12-07T21:58:03,389 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/MasterData/WALs/dbf4292a68c8,42165,1733608683017/dbf4292a68c8%2C42165%2C1733608683017.1733608683374 2024-12-07T21:58:03,389 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-07T21:58:03,389 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-07T21:58:03,390 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T21:58:03,390 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T21:58:03,390 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T21:58:03,390 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-07T21:58:03,394 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-07T21:58:03,394 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T21:58:03,395 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T21:58:03,395 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-07T21:58:03,398 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-07T21:58:03,398 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T21:58:03,398 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T21:58:03,398 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-07T21:58:03,401 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-07T21:58:03,401 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T21:58:03,401 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T21:58:03,402 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-07T21:58:03,404 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-07T21:58:03,405 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T21:58:03,405 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T21:58:03,405 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T21:58:03,405 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-07T21:58:03,405 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-07T21:58:03,406 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T21:58:03,406 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T21:58:03,406 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-07T21:58:03,406 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T21:58:03,411 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T21:58:03,411 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59608067, jitterRate=-0.11177058517932892}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-07T21:58:03,412 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733608683390Initializing all the Stores at 1733608683390Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733608683390Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733608683390Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733608683390Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733608683390Cleaning up temporary data from old regions at 1733608683406 (+16 ms)Region opened successfully at 1733608683412 (+6 ms) 2024-12-07T21:58:03,412 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-07T21:58:03,416 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3168c983, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=dbf4292a68c8/172.17.0.2:0 2024-12-07T21:58:03,417 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-07T21:58:03,417 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-07T21:58:03,417 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-07T21:58:03,417 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T21:58:03,417 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-07T21:58:03,418 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-12-07T21:58:03,418 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-07T21:58:03,419 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-07T21:58:03,420 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42165-0x1000256f48f0000, quorum=127.0.0.1:65259, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-07T21:58:03,437 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-07T21:58:03,437 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-07T21:58:03,438 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42165-0x1000256f48f0000, quorum=127.0.0.1:65259, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-07T21:58:03,445 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-07T21:58:03,445 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-07T21:58:03,447 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42165-0x1000256f48f0000, quorum=127.0.0.1:65259, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-07T21:58:03,453 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-07T21:58:03,454 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42165-0x1000256f48f0000, quorum=127.0.0.1:65259, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-07T21:58:03,462 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-07T21:58:03,464 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42165-0x1000256f48f0000, quorum=127.0.0.1:65259, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-07T21:58:03,475 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-07T21:58:03,487 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35277-0x1000256f48f0002, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-07T21:58:03,487 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42165-0x1000256f48f0000, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-07T21:58:03,487 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40365-0x1000256f48f0001, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-07T21:58:03,487 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42165-0x1000256f48f0000, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T21:58:03,487 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35277-0x1000256f48f0002, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T21:58:03,487 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40365-0x1000256f48f0001, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T21:58:03,488 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=dbf4292a68c8,42165,1733608683017, sessionid=0x1000256f48f0000, setting cluster-up flag (Was=false) 2024-12-07T21:58:03,503 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35277-0x1000256f48f0002, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T21:58:03,503 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40365-0x1000256f48f0001, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T21:58:03,503 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42165-0x1000256f48f0000, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T21:58:03,528 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-07T21:58:03,530 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=dbf4292a68c8,42165,1733608683017 2024-12-07T21:58:03,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40365-0x1000256f48f0001, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T21:58:03,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35277-0x1000256f48f0002, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T21:58:03,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42165-0x1000256f48f0000, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T21:58:03,578 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-07T21:58:03,580 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=dbf4292a68c8,42165,1733608683017 2024-12-07T21:58:03,580 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-07T21:58:03,581 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T21:58:03,582 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-07T21:58:03,582 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-07T21:58:03,582 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: dbf4292a68c8,42165,1733608683017 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-07T21:58:03,584 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/dbf4292a68c8:0, corePoolSize=5, maxPoolSize=5 2024-12-07T21:58:03,584 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/dbf4292a68c8:0, corePoolSize=5, maxPoolSize=5 2024-12-07T21:58:03,584 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/dbf4292a68c8:0, corePoolSize=5, maxPoolSize=5 2024-12-07T21:58:03,584 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/dbf4292a68c8:0, corePoolSize=5, maxPoolSize=5 2024-12-07T21:58:03,584 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/dbf4292a68c8:0, corePoolSize=10, maxPoolSize=10 2024-12-07T21:58:03,584 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:58:03,584 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/dbf4292a68c8:0, corePoolSize=2, maxPoolSize=2 2024-12-07T21:58:03,584 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:58:03,585 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733608713585 2024-12-07T21:58:03,585 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-07T21:58:03,585 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-07T21:58:03,585 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-07T21:58:03,585 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-07T21:58:03,585 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-07T21:58:03,585 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-07T21:58:03,586 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-07T21:58:03,586 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-07T21:58:03,586 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-07T21:58:03,586 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-07T21:58:03,586 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-07T21:58:03,586 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-07T21:58:03,586 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T21:58:03,586 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-07T21:58:03,586 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-07T21:58:03,587 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-07T21:58:03,587 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/dbf4292a68c8:0:becomeActiveMaster-HFileCleaner.large.0-1733608683587,5,FailOnTimeoutGroup] 2024-12-07T21:58:03,587 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/dbf4292a68c8:0:becomeActiveMaster-HFileCleaner.small.0-1733608683587,5,FailOnTimeoutGroup] 2024-12-07T21:58:03,587 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-07T21:58:03,587 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-07T21:58:03,587 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-07T21:58:03,587 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-07T21:58:03,602 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-07T21:58:03,602 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442 2024-12-07T21:58:03,607 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T21:58:03,608 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-07T21:58:03,611 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-07T21:58:03,611 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T21:58:03,611 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T21:58:03,611 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-07T21:58:03,614 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-07T21:58:03,614 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T21:58:03,614 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T21:58:03,614 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-07T21:58:03,616 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-07T21:58:03,616 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T21:58:03,617 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T21:58:03,617 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-07T21:58:03,619 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-07T21:58:03,619 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T21:58:03,619 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T21:58:03,620 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-07T21:58:03,620 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740 2024-12-07T21:58:03,620 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740 2024-12-07T21:58:03,621 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-07T21:58:03,621 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-07T21:58:03,621 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-07T21:58:03,621 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-07T21:58:03,627 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T21:58:03,627 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69759532, jitterRate=0.03949803113937378}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-07T21:58:03,628 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733608683607Initializing all the Stores at 1733608683607Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733608683607Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733608683608 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733608683608Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733608683608Cleaning up temporary data from old regions at 1733608683621 (+13 ms)Region opened successfully at 1733608683628 (+7 ms) 2024-12-07T21:58:03,628 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-07T21:58:03,628 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-07T21:58:03,628 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-07T21:58:03,628 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-07T21:58:03,628 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-07T21:58:03,629 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-07T21:58:03,629 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733608683628Disabling compacts and flushes for region at 1733608683628Disabling writes for close at 1733608683628Writing region close event to WAL at 1733608683629 (+1 ms)Closed at 1733608683629 2024-12-07T21:58:03,630 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-07T21:58:03,630 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-07T21:58:03,630 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-07T21:58:03,631 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T21:58:03,632 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-07T21:58:03,640 INFO [RS:0;dbf4292a68c8:40365 {}] regionserver.HRegionServer(746): ClusterId : 9823098d-1157-414f-951a-96aad5b923e1 2024-12-07T21:58:03,640 INFO [RS:1;dbf4292a68c8:35277 {}] regionserver.HRegionServer(746): ClusterId : 9823098d-1157-414f-951a-96aad5b923e1 2024-12-07T21:58:03,640 DEBUG [RS:1;dbf4292a68c8:35277 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-07T21:58:03,640 DEBUG [RS:0;dbf4292a68c8:40365 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-07T21:58:03,663 DEBUG [RS:1;dbf4292a68c8:35277 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-07T21:58:03,663 DEBUG [RS:0;dbf4292a68c8:40365 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-07T21:58:03,663 DEBUG [RS:0;dbf4292a68c8:40365 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-07T21:58:03,663 DEBUG [RS:1;dbf4292a68c8:35277 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-07T21:58:03,671 DEBUG [RS:1;dbf4292a68c8:35277 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-07T21:58:03,671 DEBUG [RS:0;dbf4292a68c8:40365 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-07T21:58:03,671 DEBUG [RS:1;dbf4292a68c8:35277 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@529a16ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=dbf4292a68c8/172.17.0.2:0 2024-12-07T21:58:03,671 DEBUG [RS:0;dbf4292a68c8:40365 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@82cfa47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=dbf4292a68c8/172.17.0.2:0 2024-12-07T21:58:03,688 DEBUG [RS:0;dbf4292a68c8:40365 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;dbf4292a68c8:40365 2024-12-07T21:58:03,688 DEBUG [RS:1;dbf4292a68c8:35277 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;dbf4292a68c8:35277 2024-12-07T21:58:03,688 INFO [RS:0;dbf4292a68c8:40365 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-07T21:58:03,688 INFO [RS:1;dbf4292a68c8:35277 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-07T21:58:03,688 INFO [RS:0;dbf4292a68c8:40365 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-07T21:58:03,688 INFO [RS:1;dbf4292a68c8:35277 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-07T21:58:03,688 DEBUG [RS:1;dbf4292a68c8:35277 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-07T21:58:03,688 DEBUG [RS:0;dbf4292a68c8:40365 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-07T21:58:03,689 INFO [RS:1;dbf4292a68c8:35277 {}] regionserver.HRegionServer(2659): reportForDuty to master=dbf4292a68c8,42165,1733608683017 with port=35277, startcode=1733608683217 2024-12-07T21:58:03,689 INFO [RS:0;dbf4292a68c8:40365 {}] regionserver.HRegionServer(2659): reportForDuty to master=dbf4292a68c8,42165,1733608683017 with port=40365, startcode=1733608683183 2024-12-07T21:58:03,689 DEBUG [RS:1;dbf4292a68c8:35277 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-07T21:58:03,689 DEBUG [RS:0;dbf4292a68c8:40365 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-07T21:58:03,692 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50303, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-07T21:58:03,692 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41371, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-07T21:58:03,692 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42165 {}] master.ServerManager(363): Checking decommissioned status of RegionServer dbf4292a68c8,35277,1733608683217 2024-12-07T21:58:03,692 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42165 {}] master.ServerManager(517): Registering regionserver=dbf4292a68c8,35277,1733608683217 2024-12-07T21:58:03,694 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42165 {}] master.ServerManager(363): Checking decommissioned status of RegionServer dbf4292a68c8,40365,1733608683183 2024-12-07T21:58:03,694 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42165 {}] master.ServerManager(517): Registering regionserver=dbf4292a68c8,40365,1733608683183 2024-12-07T21:58:03,694 DEBUG [RS:1;dbf4292a68c8:35277 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442 2024-12-07T21:58:03,694 DEBUG [RS:1;dbf4292a68c8:35277 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-07T21:58:03,694 DEBUG [RS:1;dbf4292a68c8:35277 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-07T21:58:03,695 DEBUG [RS:0;dbf4292a68c8:40365 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442 2024-12-07T21:58:03,695 DEBUG [RS:0;dbf4292a68c8:40365 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-07T21:58:03,695 DEBUG [RS:0;dbf4292a68c8:40365 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-07T21:58:03,703 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42165-0x1000256f48f0000, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-07T21:58:03,748 DEBUG [RS:1;dbf4292a68c8:35277 {}] zookeeper.ZKUtil(111): regionserver:35277-0x1000256f48f0002, quorum=127.0.0.1:65259, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/dbf4292a68c8,35277,1733608683217 2024-12-07T21:58:03,748 WARN [RS:1;dbf4292a68c8:35277 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-07T21:58:03,749 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [dbf4292a68c8,40365,1733608683183] 2024-12-07T21:58:03,749 INFO [RS:1;dbf4292a68c8:35277 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T21:58:03,749 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [dbf4292a68c8,35277,1733608683217] 2024-12-07T21:58:03,749 DEBUG [RS:1;dbf4292a68c8:35277 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/WALs/dbf4292a68c8,35277,1733608683217 2024-12-07T21:58:03,749 DEBUG [RS:0;dbf4292a68c8:40365 {}] zookeeper.ZKUtil(111): regionserver:40365-0x1000256f48f0001, quorum=127.0.0.1:65259, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/dbf4292a68c8,40365,1733608683183 2024-12-07T21:58:03,749 WARN [RS:0;dbf4292a68c8:40365 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-07T21:58:03,750 INFO [RS:0;dbf4292a68c8:40365 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T21:58:03,750 DEBUG [RS:0;dbf4292a68c8:40365 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/WALs/dbf4292a68c8,40365,1733608683183 2024-12-07T21:58:03,758 INFO [RS:0;dbf4292a68c8:40365 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-07T21:58:03,758 INFO [RS:1;dbf4292a68c8:35277 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-07T21:58:03,760 INFO [RS:0;dbf4292a68c8:40365 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-07T21:58:03,760 INFO [RS:0;dbf4292a68c8:40365 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-07T21:58:03,760 INFO [RS:0;dbf4292a68c8:40365 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T21:58:03,761 INFO [RS:0;dbf4292a68c8:40365 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-07T21:58:03,761 INFO [RS:0;dbf4292a68c8:40365 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-07T21:58:03,761 INFO [RS:0;dbf4292a68c8:40365 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-07T21:58:03,761 DEBUG [RS:0;dbf4292a68c8:40365 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:58:03,761 DEBUG [RS:0;dbf4292a68c8:40365 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:58:03,762 INFO [RS:1;dbf4292a68c8:35277 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-07T21:58:03,762 DEBUG [RS:0;dbf4292a68c8:40365 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:58:03,762 DEBUG [RS:0;dbf4292a68c8:40365 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:58:03,762 DEBUG [RS:0;dbf4292a68c8:40365 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:58:03,762 DEBUG [RS:0;dbf4292a68c8:40365 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/dbf4292a68c8:0, corePoolSize=2, maxPoolSize=2 2024-12-07T21:58:03,762 INFO [RS:1;dbf4292a68c8:35277 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-07T21:58:03,762 DEBUG [RS:0;dbf4292a68c8:40365 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:58:03,762 INFO [RS:1;dbf4292a68c8:35277 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T21:58:03,762 DEBUG [RS:0;dbf4292a68c8:40365 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:58:03,762 DEBUG [RS:0;dbf4292a68c8:40365 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:58:03,762 INFO [RS:1;dbf4292a68c8:35277 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-07T21:58:03,762 DEBUG [RS:0;dbf4292a68c8:40365 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:58:03,762 DEBUG [RS:0;dbf4292a68c8:40365 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:58:03,762 DEBUG [RS:0;dbf4292a68c8:40365 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:58:03,762 DEBUG [RS:0;dbf4292a68c8:40365 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/dbf4292a68c8:0, corePoolSize=3, maxPoolSize=3 2024-12-07T21:58:03,763 DEBUG [RS:0;dbf4292a68c8:40365 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/dbf4292a68c8:0, corePoolSize=3, maxPoolSize=3 2024-12-07T21:58:03,763 INFO [RS:1;dbf4292a68c8:35277 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-07T21:58:03,763 INFO [RS:1;dbf4292a68c8:35277 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-07T21:58:03,763 DEBUG [RS:1;dbf4292a68c8:35277 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:58:03,763 DEBUG [RS:1;dbf4292a68c8:35277 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:58:03,763 INFO [RS:0;dbf4292a68c8:40365 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T21:58:03,763 INFO [RS:0;dbf4292a68c8:40365 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T21:58:03,763 DEBUG [RS:1;dbf4292a68c8:35277 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:58:03,763 INFO [RS:0;dbf4292a68c8:40365 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T21:58:03,763 DEBUG [RS:1;dbf4292a68c8:35277 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:58:03,763 INFO [RS:0;dbf4292a68c8:40365 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-07T21:58:03,763 DEBUG [RS:1;dbf4292a68c8:35277 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:58:03,763 INFO [RS:0;dbf4292a68c8:40365 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-07T21:58:03,763 DEBUG [RS:1;dbf4292a68c8:35277 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/dbf4292a68c8:0, corePoolSize=2, maxPoolSize=2 2024-12-07T21:58:03,763 INFO [RS:0;dbf4292a68c8:40365 {}] hbase.ChoreService(168): Chore ScheduledChore name=dbf4292a68c8,40365,1733608683183-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-07T21:58:03,763 DEBUG [RS:1;dbf4292a68c8:35277 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:58:03,764 DEBUG [RS:1;dbf4292a68c8:35277 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:58:03,764 DEBUG [RS:1;dbf4292a68c8:35277 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:58:03,764 DEBUG [RS:1;dbf4292a68c8:35277 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:58:03,764 DEBUG [RS:1;dbf4292a68c8:35277 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:58:03,764 DEBUG [RS:1;dbf4292a68c8:35277 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:58:03,764 DEBUG [RS:1;dbf4292a68c8:35277 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/dbf4292a68c8:0, corePoolSize=3, maxPoolSize=3 2024-12-07T21:58:03,764 DEBUG [RS:1;dbf4292a68c8:35277 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/dbf4292a68c8:0, corePoolSize=3, maxPoolSize=3 2024-12-07T21:58:03,767 INFO [RS:1;dbf4292a68c8:35277 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T21:58:03,767 INFO [RS:1;dbf4292a68c8:35277 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T21:58:03,767 INFO [RS:1;dbf4292a68c8:35277 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T21:58:03,767 INFO [RS:1;dbf4292a68c8:35277 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-07T21:58:03,767 INFO [RS:1;dbf4292a68c8:35277 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-07T21:58:03,767 INFO [RS:1;dbf4292a68c8:35277 {}] hbase.ChoreService(168): Chore ScheduledChore name=dbf4292a68c8,35277,1733608683217-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-07T21:58:03,780 INFO [RS:0;dbf4292a68c8:40365 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-07T21:58:03,780 INFO [RS:0;dbf4292a68c8:40365 {}] hbase.ChoreService(168): Chore ScheduledChore name=dbf4292a68c8,40365,1733608683183-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T21:58:03,780 INFO [RS:0;dbf4292a68c8:40365 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T21:58:03,781 INFO [RS:0;dbf4292a68c8:40365 {}] regionserver.Replication(171): dbf4292a68c8,40365,1733608683183 started 2024-12-07T21:58:03,781 INFO [RS:1;dbf4292a68c8:35277 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-07T21:58:03,781 INFO [RS:1;dbf4292a68c8:35277 {}] hbase.ChoreService(168): Chore ScheduledChore name=dbf4292a68c8,35277,1733608683217-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T21:58:03,781 INFO [RS:1;dbf4292a68c8:35277 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T21:58:03,781 INFO [RS:1;dbf4292a68c8:35277 {}] regionserver.Replication(171): dbf4292a68c8,35277,1733608683217 started 2024-12-07T21:58:03,782 WARN [dbf4292a68c8:42165 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-07T21:58:03,792 INFO [RS:1;dbf4292a68c8:35277 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T21:58:03,792 INFO [RS:0;dbf4292a68c8:40365 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T21:58:03,793 INFO [RS:1;dbf4292a68c8:35277 {}] regionserver.HRegionServer(1482): Serving as dbf4292a68c8,35277,1733608683217, RpcServer on dbf4292a68c8/172.17.0.2:35277, sessionid=0x1000256f48f0002 2024-12-07T21:58:03,793 INFO [RS:0;dbf4292a68c8:40365 {}] regionserver.HRegionServer(1482): Serving as dbf4292a68c8,40365,1733608683183, RpcServer on dbf4292a68c8/172.17.0.2:40365, sessionid=0x1000256f48f0001 2024-12-07T21:58:03,793 DEBUG [RS:0;dbf4292a68c8:40365 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-07T21:58:03,793 DEBUG [RS:1;dbf4292a68c8:35277 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-07T21:58:03,793 DEBUG [RS:0;dbf4292a68c8:40365 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager dbf4292a68c8,40365,1733608683183 2024-12-07T21:58:03,793 DEBUG [RS:1;dbf4292a68c8:35277 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager dbf4292a68c8,35277,1733608683217 2024-12-07T21:58:03,793 DEBUG [RS:0;dbf4292a68c8:40365 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dbf4292a68c8,40365,1733608683183' 2024-12-07T21:58:03,793 DEBUG [RS:1;dbf4292a68c8:35277 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dbf4292a68c8,35277,1733608683217' 2024-12-07T21:58:03,793 DEBUG [RS:1;dbf4292a68c8:35277 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-07T21:58:03,793 DEBUG [RS:0;dbf4292a68c8:40365 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-07T21:58:03,794 DEBUG [RS:1;dbf4292a68c8:35277 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-07T21:58:03,794 DEBUG [RS:0;dbf4292a68c8:40365 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-07T21:58:03,794 DEBUG [RS:1;dbf4292a68c8:35277 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-07T21:58:03,794 DEBUG [RS:1;dbf4292a68c8:35277 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-07T21:58:03,794 DEBUG [RS:0;dbf4292a68c8:40365 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-07T21:58:03,794 DEBUG [RS:1;dbf4292a68c8:35277 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager dbf4292a68c8,35277,1733608683217 2024-12-07T21:58:03,794 DEBUG [RS:0;dbf4292a68c8:40365 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-07T21:58:03,794 DEBUG [RS:1;dbf4292a68c8:35277 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dbf4292a68c8,35277,1733608683217' 2024-12-07T21:58:03,794 DEBUG [RS:1;dbf4292a68c8:35277 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-07T21:58:03,794 DEBUG [RS:0;dbf4292a68c8:40365 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager dbf4292a68c8,40365,1733608683183 2024-12-07T21:58:03,794 DEBUG [RS:0;dbf4292a68c8:40365 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dbf4292a68c8,40365,1733608683183' 2024-12-07T21:58:03,794 DEBUG [RS:0;dbf4292a68c8:40365 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-07T21:58:03,795 DEBUG [RS:1;dbf4292a68c8:35277 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-07T21:58:03,795 DEBUG [RS:0;dbf4292a68c8:40365 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-07T21:58:03,795 DEBUG [RS:1;dbf4292a68c8:35277 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-07T21:58:03,795 INFO [RS:1;dbf4292a68c8:35277 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-07T21:58:03,795 INFO [RS:1;dbf4292a68c8:35277 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-07T21:58:03,795 DEBUG [RS:0;dbf4292a68c8:40365 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-07T21:58:03,795 INFO [RS:0;dbf4292a68c8:40365 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-07T21:58:03,795 INFO [RS:0;dbf4292a68c8:40365 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-07T21:58:03,897 INFO [RS:1;dbf4292a68c8:35277 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T21:58:03,897 INFO [RS:0;dbf4292a68c8:40365 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T21:58:03,898 INFO [RS:1;dbf4292a68c8:35277 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=dbf4292a68c8%2C35277%2C1733608683217, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/WALs/dbf4292a68c8,35277,1733608683217, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/oldWALs, maxLogs=55 2024-12-07T21:58:03,898 INFO [RS:0;dbf4292a68c8:40365 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=dbf4292a68c8%2C40365%2C1733608683183, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/WALs/dbf4292a68c8,40365,1733608683183, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/oldWALs, maxLogs=55 2024-12-07T21:58:03,917 INFO [RS:1;dbf4292a68c8:35277 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/WALs/dbf4292a68c8,35277,1733608683217/dbf4292a68c8%2C35277%2C1733608683217.1733608683901 2024-12-07T21:58:03,918 DEBUG [RS:1;dbf4292a68c8:35277 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-07T21:58:03,918 INFO [RS:0;dbf4292a68c8:40365 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/WALs/dbf4292a68c8,40365,1733608683183/dbf4292a68c8%2C40365%2C1733608683183.1733608683901 2024-12-07T21:58:03,918 DEBUG [RS:0;dbf4292a68c8:40365 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-07T21:58:04,032 DEBUG [dbf4292a68c8:42165 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-07T21:58:04,033 DEBUG [dbf4292a68c8:42165 {}] balancer.BalancerClusterState(204): Hosts are {dbf4292a68c8=0} racks are {/default-rack=0} 2024-12-07T21:58:04,034 DEBUG [dbf4292a68c8:42165 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-07T21:58:04,034 DEBUG [dbf4292a68c8:42165 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-07T21:58:04,034 DEBUG [dbf4292a68c8:42165 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-07T21:58:04,034 DEBUG [dbf4292a68c8:42165 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-07T21:58:04,034 INFO [dbf4292a68c8:42165 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-07T21:58:04,034 INFO [dbf4292a68c8:42165 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-07T21:58:04,034 DEBUG [dbf4292a68c8:42165 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-07T21:58:04,035 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=dbf4292a68c8,40365,1733608683183 2024-12-07T21:58:04,037 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as dbf4292a68c8,40365,1733608683183, state=OPENING 2024-12-07T21:58:04,075 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-07T21:58:04,131 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35277-0x1000256f48f0002, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T21:58:04,131 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40365-0x1000256f48f0001, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T21:58:04,131 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42165-0x1000256f48f0000, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T21:58:04,133 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T21:58:04,133 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T21:58:04,133 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T21:58:04,133 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=dbf4292a68c8,40365,1733608683183}] 2024-12-07T21:58:04,133 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T21:58:04,287 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-07T21:58:04,293 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60041, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-07T21:58:04,297 INFO [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-07T21:58:04,297 INFO [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T21:58:04,298 INFO [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-07T21:58:04,299 INFO [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=dbf4292a68c8%2C40365%2C1733608683183.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/WALs/dbf4292a68c8,40365,1733608683183, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/oldWALs, maxLogs=55 2024-12-07T21:58:04,317 INFO [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/WALs/dbf4292a68c8,40365,1733608683183/dbf4292a68c8%2C40365%2C1733608683183.meta.1733608684300.meta 2024-12-07T21:58:04,317 DEBUG [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-07T21:58:04,318 DEBUG [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-07T21:58:04,318 DEBUG [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-07T21:58:04,318 DEBUG [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-07T21:58:04,318 INFO [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-07T21:58:04,318 DEBUG [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-07T21:58:04,318 DEBUG [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T21:58:04,319 DEBUG [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-07T21:58:04,319 DEBUG [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-07T21:58:04,320 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-07T21:58:04,321 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-07T21:58:04,321 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T21:58:04,321 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T21:58:04,321 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-07T21:58:04,322 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-07T21:58:04,322 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T21:58:04,322 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T21:58:04,322 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-07T21:58:04,322 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-07T21:58:04,322 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T21:58:04,322 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T21:58:04,323 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-07T21:58:04,323 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-07T21:58:04,323 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T21:58:04,323 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T21:58:04,323 DEBUG [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-07T21:58:04,324 DEBUG [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740 2024-12-07T21:58:04,324 DEBUG [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740 2024-12-07T21:58:04,324 DEBUG [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-07T21:58:04,324 DEBUG [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-07T21:58:04,324 DEBUG [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-07T21:58:04,325 DEBUG [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-07T21:58:04,325 INFO [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63190186, jitterRate=-0.058392852544784546}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-07T21:58:04,325 DEBUG [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-07T21:58:04,326 DEBUG [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733608684319Writing region info on filesystem at 1733608684319Initializing all the Stores at 1733608684319Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733608684319Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733608684320 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733608684320Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733608684320Cleaning up temporary data from old regions at 1733608684324 (+4 ms)Running coprocessor post-open hooks at 1733608684326 (+2 ms)Region opened successfully at 1733608684326 2024-12-07T21:58:04,326 INFO [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733608684286 2024-12-07T21:58:04,329 DEBUG [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-07T21:58:04,329 INFO [RS_OPEN_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-07T21:58:04,329 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=dbf4292a68c8,40365,1733608683183 2024-12-07T21:58:04,330 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as dbf4292a68c8,40365,1733608683183, state=OPEN 2024-12-07T21:58:04,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42165-0x1000256f48f0000, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T21:58:04,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40365-0x1000256f48f0001, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T21:58:04,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35277-0x1000256f48f0002, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T21:58:04,345 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=dbf4292a68c8,40365,1733608683183 2024-12-07T21:58:04,345 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T21:58:04,345 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T21:58:04,345 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T21:58:04,348 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-07T21:58:04,348 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=dbf4292a68c8,40365,1733608683183 in 212 msec 2024-12-07T21:58:04,350 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-07T21:58:04,351 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 718 msec 2024-12-07T21:58:04,351 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-07T21:58:04,351 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-07T21:58:04,355 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T21:58:04,355 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=dbf4292a68c8,40365,1733608683183, seqNum=-1] 2024-12-07T21:58:04,355 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T21:58:04,357 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47259, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T21:58:04,364 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 781 msec 2024-12-07T21:58:04,364 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733608684364, completionTime=-1 2024-12-07T21:58:04,364 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-07T21:58:04,365 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T21:58:04,367 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T21:58:04,367 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733608744367 2024-12-07T21:58:04,367 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733608804367 2024-12-07T21:58:04,367 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-12-07T21:58:04,368 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-07T21:58:04,368 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dbf4292a68c8,42165,1733608683017-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T21:58:04,368 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dbf4292a68c8,42165,1733608683017-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T21:58:04,369 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dbf4292a68c8,42165,1733608683017-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T21:58:04,369 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-dbf4292a68c8:42165, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T21:58:04,369 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-07T21:58:04,369 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-07T21:58:04,372 DEBUG [master/dbf4292a68c8:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-07T21:58:04,374 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.095sec 2024-12-07T21:58:04,374 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-07T21:58:04,374 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-07T21:58:04,375 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-07T21:58:04,375 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-07T21:58:04,375 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-07T21:58:04,375 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dbf4292a68c8,42165,1733608683017-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-07T21:58:04,375 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dbf4292a68c8,42165,1733608683017-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-07T21:58:04,376 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-07T21:58:04,376 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-07T21:58:04,376 INFO [master/dbf4292a68c8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dbf4292a68c8,42165,1733608683017-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T21:58:04,441 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2129cce9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T21:58:04,441 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request dbf4292a68c8,42165,-1 for getting cluster id 2024-12-07T21:58:04,442 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T21:58:04,446 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9823098d-1157-414f-951a-96aad5b923e1' 2024-12-07T21:58:04,447 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T21:58:04,447 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9823098d-1157-414f-951a-96aad5b923e1" 2024-12-07T21:58:04,448 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62bf8d9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T21:58:04,448 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [dbf4292a68c8,42165,-1] 2024-12-07T21:58:04,449 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T21:58:04,449 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T21:58:04,451 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46976, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T21:58:04,452 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e62305b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T21:58:04,453 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T21:58:04,454 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=dbf4292a68c8,40365,1733608683183, seqNum=-1] 2024-12-07T21:58:04,454 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T21:58:04,456 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54690, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T21:58:04,459 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=dbf4292a68c8,42165,1733608683017 2024-12-07T21:58:04,459 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-07T21:58:04,460 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncConnectionImpl(321): The fetched master address is dbf4292a68c8,42165,1733608683017 2024-12-07T21:58:04,461 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3676468b 2024-12-07T21:58:04,461 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-07T21:58:04,463 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46982, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-07T21:58:04,464 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42165 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1733608684459', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-07T21:58:04,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42165 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733608684459 2024-12-07T21:58:04,465 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733608684459 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-07T21:58:04,465 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T21:58:04,466 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42165 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1733608684459" procId is: 4 2024-12-07T21:58:04,467 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733608684459 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-07T21:58:04,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T21:58:04,480 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733608684459-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => dfaeddf40c94f74d5082b9efaad353be, NAME => 'testBlockEvictionOnGracefulStop1733608684459,,1733608684463.dfaeddf40c94f74d5082b9efaad353be.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1733608684459', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442 2024-12-07T21:58:04,485 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733608684459-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733608684459,,1733608684463.dfaeddf40c94f74d5082b9efaad353be.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T21:58:04,485 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733608684459-pool-0 {}] regionserver.HRegion(1722): Closing dfaeddf40c94f74d5082b9efaad353be, disabling compactions & flushes 2024-12-07T21:58:04,485 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733608684459-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733608684459,,1733608684463.dfaeddf40c94f74d5082b9efaad353be. 2024-12-07T21:58:04,486 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733608684459-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733608684459,,1733608684463.dfaeddf40c94f74d5082b9efaad353be. 2024-12-07T21:58:04,486 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733608684459-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733608684459,,1733608684463.dfaeddf40c94f74d5082b9efaad353be. after waiting 0 ms 2024-12-07T21:58:04,486 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733608684459-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733608684459,,1733608684463.dfaeddf40c94f74d5082b9efaad353be. 2024-12-07T21:58:04,486 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733608684459-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733608684459,,1733608684463.dfaeddf40c94f74d5082b9efaad353be. 2024-12-07T21:58:04,486 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733608684459-pool-0 {}] regionserver.HRegion(1676): Region close journal for dfaeddf40c94f74d5082b9efaad353be: Waiting for close lock at 1733608684485Disabling compacts and flushes for region at 1733608684485Disabling writes for close at 1733608684486 (+1 ms)Writing region close event to WAL at 1733608684486Closed at 1733608684486 2024-12-07T21:58:04,487 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733608684459 execute state=CREATE_TABLE_ADD_TO_META 2024-12-07T21:58:04,487 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1733608684459,,1733608684463.dfaeddf40c94f74d5082b9efaad353be.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1733608684487"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733608684487"}]},"ts":"1733608684487"} 2024-12-07T21:58:04,490 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-07T21:58:04,491 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733608684459 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-07T21:58:04,491 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1733608684459","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733608684491"}]},"ts":"1733608684491"} 2024-12-07T21:58:04,494 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1733608684459, state=ENABLING in hbase:meta 2024-12-07T21:58:04,494 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {dbf4292a68c8=0} racks are {/default-rack=0} 2024-12-07T21:58:04,495 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-07T21:58:04,495 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-07T21:58:04,495 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-07T21:58:04,495 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-07T21:58:04,495 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-07T21:58:04,495 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-07T21:58:04,495 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-07T21:58:04,495 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733608684459, region=dfaeddf40c94f74d5082b9efaad353be, ASSIGN}] 2024-12-07T21:58:04,496 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733608684459, region=dfaeddf40c94f74d5082b9efaad353be, ASSIGN 2024-12-07T21:58:04,497 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733608684459, region=dfaeddf40c94f74d5082b9efaad353be, ASSIGN; state=OFFLINE, location=dbf4292a68c8,35277,1733608683217; forceNewPlan=false, retain=false 2024-12-07T21:58:04,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T21:58:04,648 INFO [dbf4292a68c8:42165 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-07T21:58:04,649 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=dfaeddf40c94f74d5082b9efaad353be, regionState=OPENING, regionLocation=dbf4292a68c8,35277,1733608683217 2024-12-07T21:58:04,654 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733608684459, region=dfaeddf40c94f74d5082b9efaad353be, ASSIGN because future has completed 2024-12-07T21:58:04,655 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure dfaeddf40c94f74d5082b9efaad353be, server=dbf4292a68c8,35277,1733608683217}] 2024-12-07T21:58:04,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T21:58:04,809 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-07T21:58:04,812 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44335, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-07T21:58:04,816 INFO [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1733608684459,,1733608684463.dfaeddf40c94f74d5082b9efaad353be. 2024-12-07T21:58:04,816 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => dfaeddf40c94f74d5082b9efaad353be, NAME => 'testBlockEvictionOnGracefulStop1733608684459,,1733608684463.dfaeddf40c94f74d5082b9efaad353be.', STARTKEY => '', ENDKEY => ''} 2024-12-07T21:58:04,817 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1733608684459 dfaeddf40c94f74d5082b9efaad353be 2024-12-07T21:58:04,818 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733608684459,,1733608684463.dfaeddf40c94f74d5082b9efaad353be.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T21:58:04,818 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for dfaeddf40c94f74d5082b9efaad353be 2024-12-07T21:58:04,818 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for dfaeddf40c94f74d5082b9efaad353be 2024-12-07T21:58:04,820 INFO [StoreOpener-dfaeddf40c94f74d5082b9efaad353be-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region dfaeddf40c94f74d5082b9efaad353be 2024-12-07T21:58:04,825 INFO [StoreOpener-dfaeddf40c94f74d5082b9efaad353be-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dfaeddf40c94f74d5082b9efaad353be columnFamilyName family 2024-12-07T21:58:04,825 DEBUG [StoreOpener-dfaeddf40c94f74d5082b9efaad353be-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T21:58:04,825 INFO [StoreOpener-dfaeddf40c94f74d5082b9efaad353be-1 {}] regionserver.HStore(327): Store=dfaeddf40c94f74d5082b9efaad353be/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T21:58:04,825 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for dfaeddf40c94f74d5082b9efaad353be 2024-12-07T21:58:04,826 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/default/testBlockEvictionOnGracefulStop1733608684459/dfaeddf40c94f74d5082b9efaad353be 2024-12-07T21:58:04,826 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/default/testBlockEvictionOnGracefulStop1733608684459/dfaeddf40c94f74d5082b9efaad353be 2024-12-07T21:58:04,827 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for dfaeddf40c94f74d5082b9efaad353be 2024-12-07T21:58:04,827 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for dfaeddf40c94f74d5082b9efaad353be 2024-12-07T21:58:04,828 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for dfaeddf40c94f74d5082b9efaad353be 2024-12-07T21:58:04,834 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/default/testBlockEvictionOnGracefulStop1733608684459/dfaeddf40c94f74d5082b9efaad353be/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T21:58:04,835 INFO [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened dfaeddf40c94f74d5082b9efaad353be; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60891451, jitterRate=-0.09264667332172394}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T21:58:04,835 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dfaeddf40c94f74d5082b9efaad353be 2024-12-07T21:58:04,836 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for dfaeddf40c94f74d5082b9efaad353be: Running coprocessor pre-open hook at 1733608684819Writing region info on filesystem at 1733608684819Initializing all the Stores at 1733608684819Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733608684819Cleaning up temporary data from old regions at 1733608684827 (+8 ms)Running coprocessor post-open hooks at 1733608684835 (+8 ms)Region opened successfully at 1733608684836 (+1 ms) 2024-12-07T21:58:04,837 INFO [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1733608684459,,1733608684463.dfaeddf40c94f74d5082b9efaad353be., pid=6, masterSystemTime=1733608684808 2024-12-07T21:58:04,840 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1733608684459,,1733608684463.dfaeddf40c94f74d5082b9efaad353be. 2024-12-07T21:58:04,840 INFO [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1733608684459,,1733608684463.dfaeddf40c94f74d5082b9efaad353be. 2024-12-07T21:58:04,841 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=dfaeddf40c94f74d5082b9efaad353be, regionState=OPEN, openSeqNum=2, regionLocation=dbf4292a68c8,35277,1733608683217 2024-12-07T21:58:04,844 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure dfaeddf40c94f74d5082b9efaad353be, server=dbf4292a68c8,35277,1733608683217 because future has completed 2024-12-07T21:58:04,847 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-07T21:58:04,848 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure dfaeddf40c94f74d5082b9efaad353be, server=dbf4292a68c8,35277,1733608683217 in 190 msec 2024-12-07T21:58:04,849 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-07T21:58:04,849 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733608684459, region=dfaeddf40c94f74d5082b9efaad353be, ASSIGN in 352 msec 2024-12-07T21:58:04,849 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733608684459 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-07T21:58:04,849 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1733608684459","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733608684849"}]},"ts":"1733608684849"} 2024-12-07T21:58:04,851 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1733608684459, state=ENABLED in hbase:meta 2024-12-07T21:58:04,852 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733608684459 execute state=CREATE_TABLE_POST_OPERATION 2024-12-07T21:58:04,853 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733608684459 in 388 msec 2024-12-07T21:58:05,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42165 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T21:58:05,099 INFO [RPCClient-NioEventLoopGroup-5-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1733608684459 completed 2024-12-07T21:58:05,099 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1733608684459 get assigned. Timeout = 60000ms 2024-12-07T21:58:05,099 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-07T21:58:05,103 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1733608684459 assigned to meta. Checking AM states. 2024-12-07T21:58:05,103 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-07T21:58:05,103 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1733608684459 assigned. 2024-12-07T21:58:05,107 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1733608684459', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1733608684459,,1733608684463.dfaeddf40c94f74d5082b9efaad353be., hostname=dbf4292a68c8,35277,1733608683217, seqNum=2] 2024-12-07T21:58:05,108 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T21:58:05,114 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44420, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T21:58:05,118 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing dfaeddf40c94f74d5082b9efaad353be 1/1 column families, dataSize=78 B heapSize=480 B 2024-12-07T21:58:05,126 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 64ca6b6064e54c399853fdae078cb99d 2024-12-07T21:58:05,144 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/default/testBlockEvictionOnGracefulStop1733608684459/dfaeddf40c94f74d5082b9efaad353be/.tmp/family/64ca6b6064e54c399853fdae078cb99d is 43, key is row1/family:qf1/1/Put/seqid=0 2024-12-07T21:58:05,145 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/default/testBlockEvictionOnGracefulStop1733608684459/dfaeddf40c94f74d5082b9efaad353be/.tmp/family/64ca6b6064e54c399853fdae078cb99d 2024-12-07T21:58:05,148 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/default/testBlockEvictionOnGracefulStop1733608684459/dfaeddf40c94f74d5082b9efaad353be/.tmp/family/64ca6b6064e54c399853fdae078cb99d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/default/testBlockEvictionOnGracefulStop1733608684459/dfaeddf40c94f74d5082b9efaad353be/family/64ca6b6064e54c399853fdae078cb99d 2024-12-07T21:58:05,150 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/default/testBlockEvictionOnGracefulStop1733608684459/dfaeddf40c94f74d5082b9efaad353be/family/64ca6b6064e54c399853fdae078cb99d, delay=915 ms 2024-12-07T21:58:05,151 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/default/testBlockEvictionOnGracefulStop1733608684459/dfaeddf40c94f74d5082b9efaad353be/family/64ca6b6064e54c399853fdae078cb99d, entries=2, sequenceid=6, filesize=4.9 K 2024-12-07T21:58:05,152 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for dfaeddf40c94f74d5082b9efaad353be in 34ms, sequenceid=6, compaction requested=false 2024-12-07T21:58:05,153 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1733608684459' 2024-12-07T21:58:05,153 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for dfaeddf40c94f74d5082b9efaad353be: 2024-12-07T21:58:05,195 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-07T21:58:06,066 DEBUG [hfile-prefetch-1733608682350 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 64ca6b6064e54c399853fdae078cb99d_0. 2024-12-07T21:58:06,067 DEBUG [hfile-prefetch-1733608682350 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 64ca6b6064e54c399853fdae078cb99d_117. 2024-12-07T21:58:06,067 DEBUG [hfile-prefetch-1733608682350 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/default/testBlockEvictionOnGracefulStop1733608684459/dfaeddf40c94f74d5082b9efaad353be/family/64ca6b6064e54c399853fdae078cb99d, with total blocks 2, and data blocks 1 2024-12-07T21:58:06,067 DEBUG [hfile-prefetch-1733608682350 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-07T21:58:06,067 DEBUG [hfile-prefetch-1733608682350 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/default/testBlockEvictionOnGracefulStop1733608684459/dfaeddf40c94f74d5082b9efaad353be/family/64ca6b6064e54c399853fdae078cb99d, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-12-07T21:58:06,067 DEBUG [hfile-prefetch-1733608682350 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/default/testBlockEvictionOnGracefulStop1733608684459/dfaeddf40c94f74d5082b9efaad353be/family/64ca6b6064e54c399853fdae078cb99d. Will sleep 100ms and try the verification again. 2024-12-07T21:58:06,154 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;dbf4292a68c8:35277,5,FailOnTimeoutGroup] 2024-12-07T21:58:06,154 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'dbf4292a68c8,35277,1733608683217' ***** 2024-12-07T21:58:06,154 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-12-07T21:58:06,154 INFO [RS:1;dbf4292a68c8:35277 {}] bucket.BucketCache(1738): Disabling cache 2024-12-07T21:58:06,154 INFO [RS:1;dbf4292a68c8:35277 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05a8dc90-5822-7fba-c9a9-13ddf09bc5ac/bucket.persistence 2024-12-07T21:58:06,154 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-07T21:58:06,154 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-07T21:58:06,154 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-07T21:58:06,154 INFO [RS:1;dbf4292a68c8:35277 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-07T21:58:06,155 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-07T21:58:06,165 DEBUG [RS:1;dbf4292a68c8:35277 {}] bucket.BucketCache(1487): Thread RS:1;dbf4292a68c8:35277 started persisting bucket cache to file 2024-12-07T21:58:06,165 DEBUG [RS:1;dbf4292a68c8:35277 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-07T21:58:06,165 DEBUG [RS:1;dbf4292a68c8:35277 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-12-07T21:58:06,167 DEBUG [hfile-prefetch-1733608682350 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/default/testBlockEvictionOnGracefulStop1733608684459/dfaeddf40c94f74d5082b9efaad353be/family/64ca6b6064e54c399853fdae078cb99d, with total blocks 2, and data blocks 1 2024-12-07T21:58:06,168 DEBUG [hfile-prefetch-1733608682350 {}] bucket.BucketCache(2285): iterating over 1 entries in the backing map 2024-12-07T21:58:06,168 DEBUG [hfile-prefetch-1733608682350 {}] bucket.BucketCache(2293): found block for file 64ca6b6064e54c399853fdae078cb99d in the backing map. Acquiring read lock for offset 0 2024-12-07T21:58:06,168 DEBUG [RS:1;dbf4292a68c8:35277 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-12-07T21:58:06,168 DEBUG [RS:1;dbf4292a68c8:35277 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1733608686165 2024-12-07T21:58:06,168 DEBUG [hfile-prefetch-1733608682350 {}] bucket.BucketCache(2305): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/default/testBlockEvictionOnGracefulStop1733608684459/dfaeddf40c94f74d5082b9efaad353be/family/64ca6b6064e54c399853fdae078cb99d has now been fully cached. 2024-12-07T21:58:06,168 DEBUG [RS:1;dbf4292a68c8:35277 {}] bucket.BucketCache(1509): Thread RS:1;dbf4292a68c8:35277 finished persisting bucket cache to file, renaming 2024-12-07T21:58:06,168 INFO [RS:1;dbf4292a68c8:35277 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-07T21:58:06,168 DEBUG [hfile-prefetch-1733608682350 {}] hfile.PrefetchExecutor(121): Prefetch completed for 64ca6b6064e54c399853fdae078cb99d 2024-12-07T21:58:06,168 INFO [RS:1;dbf4292a68c8:35277 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-07T21:58:06,168 INFO [RS:1;dbf4292a68c8:35277 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-07T21:58:06,168 INFO [RS:1;dbf4292a68c8:35277 {}] regionserver.HRegionServer(3091): Received CLOSE for dfaeddf40c94f74d5082b9efaad353be 2024-12-07T21:58:06,168 INFO [RS:1;dbf4292a68c8:35277 {}] regionserver.HRegionServer(959): stopping server dbf4292a68c8,35277,1733608683217 2024-12-07T21:58:06,168 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-07T21:58:06,169 INFO [RS:1;dbf4292a68c8:35277 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-07T21:58:06,169 INFO [RS:1;dbf4292a68c8:35277 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;dbf4292a68c8:35277. 2024-12-07T21:58:06,169 INFO [RS_CLOSE_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1733608684459,,1733608684463.dfaeddf40c94f74d5082b9efaad353be. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-07T21:58:06,169 DEBUG [RS_CLOSE_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing dfaeddf40c94f74d5082b9efaad353be, disabling compactions & flushes 2024-12-07T21:58:06,169 DEBUG [RS:1;dbf4292a68c8:35277 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T21:58:06,169 INFO [RS_CLOSE_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733608684459,,1733608684463.dfaeddf40c94f74d5082b9efaad353be. 2024-12-07T21:58:06,169 DEBUG [RS:1;dbf4292a68c8:35277 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T21:58:06,169 DEBUG [RS_CLOSE_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733608684459,,1733608684463.dfaeddf40c94f74d5082b9efaad353be. 2024-12-07T21:58:06,169 INFO [RS:1;dbf4292a68c8:35277 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-07T21:58:06,169 DEBUG [RS_CLOSE_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733608684459,,1733608684463.dfaeddf40c94f74d5082b9efaad353be. after waiting 0 ms 2024-12-07T21:58:06,169 DEBUG [RS:1;dbf4292a68c8:35277 {}] regionserver.HRegionServer(1325): Online Regions={dfaeddf40c94f74d5082b9efaad353be=testBlockEvictionOnGracefulStop1733608684459,,1733608684463.dfaeddf40c94f74d5082b9efaad353be.} 2024-12-07T21:58:06,169 DEBUG [RS_CLOSE_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733608684459,,1733608684463.dfaeddf40c94f74d5082b9efaad353be. 2024-12-07T21:58:06,169 DEBUG [RS:1;dbf4292a68c8:35277 {}] regionserver.HRegionServer(1351): Waiting on dfaeddf40c94f74d5082b9efaad353be 2024-12-07T21:58:06,170 INFO [regionserver/dbf4292a68c8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T21:58:06,174 DEBUG [RS_CLOSE_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/default/testBlockEvictionOnGracefulStop1733608684459/dfaeddf40c94f74d5082b9efaad353be/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-07T21:58:06,174 INFO [RS_CLOSE_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733608684459,,1733608684463.dfaeddf40c94f74d5082b9efaad353be. 2024-12-07T21:58:06,175 DEBUG [RS_CLOSE_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for dfaeddf40c94f74d5082b9efaad353be: Waiting for close lock at 1733608686169Running coprocessor pre-close hooks at 1733608686169Disabling compacts and flushes for region at 1733608686169Disabling writes for close at 1733608686169Writing region close event to WAL at 1733608686170 (+1 ms)Running coprocessor post-close hooks at 1733608686174 (+4 ms)Closed at 1733608686174 2024-12-07T21:58:06,175 DEBUG [RS_CLOSE_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1733608684459,,1733608684463.dfaeddf40c94f74d5082b9efaad353be. 2024-12-07T21:58:06,200 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-07T21:58:06,200 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-07T21:58:06,200 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-07T21:58:06,200 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-07T21:58:06,203 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-07T21:58:06,203 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733608686200 2024-12-07T21:58:06,203 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-07T21:58:06,369 INFO [RS:1;dbf4292a68c8:35277 {}] regionserver.HRegionServer(976): stopping server dbf4292a68c8,35277,1733608683217; all regions closed. 2024-12-07T21:58:06,372 DEBUG [RS:1;dbf4292a68c8:35277 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/oldWALs 2024-12-07T21:58:06,372 INFO [RS:1;dbf4292a68c8:35277 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL dbf4292a68c8%2C35277%2C1733608683217:(num 1733608683901) 2024-12-07T21:58:06,372 DEBUG [RS:1;dbf4292a68c8:35277 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T21:58:06,372 INFO [RS:1;dbf4292a68c8:35277 {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T21:58:06,372 INFO [RS:1;dbf4292a68c8:35277 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-07T21:58:06,372 INFO [RS:1;dbf4292a68c8:35277 {}] hbase.ChoreService(370): Chore service for: regionserver/dbf4292a68c8:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-07T21:58:06,372 INFO [RS:1;dbf4292a68c8:35277 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-07T21:58:06,372 INFO [RS:1;dbf4292a68c8:35277 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-07T21:58:06,372 INFO [regionserver/dbf4292a68c8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-07T21:58:06,372 INFO [RS:1;dbf4292a68c8:35277 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-07T21:58:06,372 INFO [RS:1;dbf4292a68c8:35277 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-07T21:58:06,373 INFO [RS:1;dbf4292a68c8:35277 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35277 2024-12-07T21:58:06,399 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42165-0x1000256f48f0000, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-07T21:58:06,399 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35277-0x1000256f48f0002, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/dbf4292a68c8,35277,1733608683217 2024-12-07T21:58:06,399 INFO [RS:1;dbf4292a68c8:35277 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-07T21:58:06,408 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [dbf4292a68c8,35277,1733608683217] 2024-12-07T21:58:06,416 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/dbf4292a68c8,35277,1733608683217 already deleted, retry=false 2024-12-07T21:58:06,416 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of dbf4292a68c8,35277,1733608683217 on dbf4292a68c8,42165,1733608683017 2024-12-07T21:58:06,421 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure dbf4292a68c8,35277,1733608683217, splitWal=true, meta=false 2024-12-07T21:58:06,424 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure dbf4292a68c8,35277,1733608683217, splitWal=true, meta=false 2024-12-07T21:58:06,424 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for dbf4292a68c8,35277,1733608683217 (carryingMeta=false) dbf4292a68c8,35277,1733608683217/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@2676a798[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-07T21:58:06,425 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): dbf4292a68c8,35277,1733608683217 had 1 regions 2024-12-07T21:58:06,426 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure dbf4292a68c8,35277,1733608683217, splitWal=true, meta=false, isMeta: false 2024-12-07T21:58:06,427 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/WALs/dbf4292a68c8,35277,1733608683217-splitting 2024-12-07T21:58:06,427 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/WALs/dbf4292a68c8,35277,1733608683217-splitting dir is empty, no logs to split. 2024-12-07T21:58:06,427 INFO [PEWorker-1 {}] master.SplitWALManager(105): dbf4292a68c8,35277,1733608683217 WAL count=0, meta=false 2024-12-07T21:58:06,432 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/WALs/dbf4292a68c8,35277,1733608683217-splitting dir is empty, no logs to split. 2024-12-07T21:58:06,432 INFO [PEWorker-1 {}] master.SplitWALManager(105): dbf4292a68c8,35277,1733608683217 WAL count=0, meta=false 2024-12-07T21:58:06,432 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if dbf4292a68c8,35277,1733608683217 WAL splitting is done? wals=0, meta=false 2024-12-07T21:58:06,433 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for dbf4292a68c8,35277,1733608683217 failed, ignore...File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/WALs/dbf4292a68c8,35277,1733608683217-splitting does not exist 2024-12-07T21:58:06,436 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733608684459, region=dfaeddf40c94f74d5082b9efaad353be, ASSIGN}] 2024-12-07T21:58:06,437 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733608684459, region=dfaeddf40c94f74d5082b9efaad353be, ASSIGN 2024-12-07T21:58:06,438 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733608684459, region=dfaeddf40c94f74d5082b9efaad353be, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-07T21:58:06,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35277-0x1000256f48f0002, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T21:58:06,508 INFO [RS:1;dbf4292a68c8:35277 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-07T21:58:06,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35277-0x1000256f48f0002, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T21:58:06,508 INFO [RS:1;dbf4292a68c8:35277 {}] regionserver.HRegionServer(1031): Exiting; stopping=dbf4292a68c8,35277,1733608683217; zookeeper connection closed. 2024-12-07T21:58:06,508 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@761c6155 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@761c6155 2024-12-07T21:58:06,589 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=dfaeddf40c94f74d5082b9efaad353be, regionState=OPENING, regionLocation=dbf4292a68c8,40365,1733608683183 2024-12-07T21:58:06,592 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733608684459, region=dfaeddf40c94f74d5082b9efaad353be, ASSIGN because future has completed 2024-12-07T21:58:06,592 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure dfaeddf40c94f74d5082b9efaad353be, server=dbf4292a68c8,40365,1733608683183}] 2024-12-07T21:58:06,669 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/dbf4292a68c8:0 server-side Connection retries=45 2024-12-07T21:58:06,669 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T21:58:06,669 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-07T21:58:06,669 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-07T21:58:06,669 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T21:58:06,669 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-07T21:58:06,669 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-07T21:58:06,669 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-07T21:58:06,670 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33333 2024-12-07T21:58:06,671 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33333 connecting to ZooKeeper ensemble=127.0.0.1:65259 2024-12-07T21:58:06,671 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T21:58:06,671 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T21:58:06,691 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:333330x0, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-07T21:58:06,692 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:333330x0, quorum=127.0.0.1:65259, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-12-07T21:58:06,692 DEBUG [pool-297-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-12-07T21:58:06,692 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33333-0x1000256f48f0003 connected 2024-12-07T21:58:06,692 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-07T21:58:06,693 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05a8dc90-5822-7fba-c9a9-13ddf09bc5ac/bucket.cache 2024-12-07T21:58:06,693 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-07T21:58:06,693 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-07T21:58:06,693 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-07T21:58:06,694 INFO [Thread-202 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-07T21:58:06,694 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05a8dc90-5822-7fba-c9a9-13ddf09bc5ac/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05a8dc90-5822-7fba-c9a9-13ddf09bc5ac/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-07T21:58:06,694 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-07T21:58:06,695 INFO [Thread-202 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-12-07T21:58:06,695 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:33333-0x1000256f48f0003, quorum=127.0.0.1:65259, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-07T21:58:06,695 INFO [Thread-202 {}] bucket.BucketCache(1705): Retrieved 0 of chunks with blockCount = 0. 2024-12-07T21:58:06,699 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33333-0x1000256f48f0003, quorum=127.0.0.1:65259, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-07T21:58:06,703 DEBUG [Thread-202 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-12-07T21:58:06,704 INFO [Thread-202 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-07T21:58:06,704 INFO [Thread-202 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 0 2024-12-07T21:58:06,704 INFO [Thread-202 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05a8dc90-5822-7fba-c9a9-13ddf09bc5ac/bucket.persistence is complete. 2024-12-07T21:58:06,709 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33333 2024-12-07T21:58:06,712 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33333 2024-12-07T21:58:06,718 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33333 2024-12-07T21:58:06,720 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33333 2024-12-07T21:58:06,724 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33333 2024-12-07T21:58:06,731 INFO [RS:2;dbf4292a68c8:33333 {}] regionserver.HRegionServer(746): ClusterId : 9823098d-1157-414f-951a-96aad5b923e1 2024-12-07T21:58:06,731 DEBUG [RS:2;dbf4292a68c8:33333 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-07T21:58:06,744 DEBUG [RS:2;dbf4292a68c8:33333 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-07T21:58:06,744 DEBUG [RS:2;dbf4292a68c8:33333 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-07T21:58:06,749 INFO [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1733608684459,,1733608684463.dfaeddf40c94f74d5082b9efaad353be. 2024-12-07T21:58:06,750 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => dfaeddf40c94f74d5082b9efaad353be, NAME => 'testBlockEvictionOnGracefulStop1733608684459,,1733608684463.dfaeddf40c94f74d5082b9efaad353be.', STARTKEY => '', ENDKEY => ''} 2024-12-07T21:58:06,750 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1733608684459 dfaeddf40c94f74d5082b9efaad353be 2024-12-07T21:58:06,750 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733608684459,,1733608684463.dfaeddf40c94f74d5082b9efaad353be.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T21:58:06,750 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for dfaeddf40c94f74d5082b9efaad353be 2024-12-07T21:58:06,751 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for dfaeddf40c94f74d5082b9efaad353be 2024-12-07T21:58:06,754 DEBUG [RS:2;dbf4292a68c8:33333 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-07T21:58:06,754 DEBUG [RS:2;dbf4292a68c8:33333 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37ad5cd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=dbf4292a68c8/172.17.0.2:0 2024-12-07T21:58:06,755 INFO [StoreOpener-dfaeddf40c94f74d5082b9efaad353be-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region dfaeddf40c94f74d5082b9efaad353be 2024-12-07T21:58:06,759 INFO [StoreOpener-dfaeddf40c94f74d5082b9efaad353be-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dfaeddf40c94f74d5082b9efaad353be columnFamilyName family 2024-12-07T21:58:06,760 DEBUG [StoreOpener-dfaeddf40c94f74d5082b9efaad353be-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T21:58:06,764 DEBUG [StoreFileOpener-dfaeddf40c94f74d5082b9efaad353be-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/default/testBlockEvictionOnGracefulStop1733608684459/dfaeddf40c94f74d5082b9efaad353be/family/64ca6b6064e54c399853fdae078cb99d, delay=921 ms 2024-12-07T21:58:06,766 DEBUG [StoreOpener-dfaeddf40c94f74d5082b9efaad353be-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/default/testBlockEvictionOnGracefulStop1733608684459/dfaeddf40c94f74d5082b9efaad353be/family/64ca6b6064e54c399853fdae078cb99d 2024-12-07T21:58:06,766 INFO [StoreOpener-dfaeddf40c94f74d5082b9efaad353be-1 {}] regionserver.HStore(327): Store=dfaeddf40c94f74d5082b9efaad353be/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T21:58:06,768 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for dfaeddf40c94f74d5082b9efaad353be 2024-12-07T21:58:06,769 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/default/testBlockEvictionOnGracefulStop1733608684459/dfaeddf40c94f74d5082b9efaad353be 2024-12-07T21:58:06,770 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/default/testBlockEvictionOnGracefulStop1733608684459/dfaeddf40c94f74d5082b9efaad353be 2024-12-07T21:58:06,770 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for dfaeddf40c94f74d5082b9efaad353be 2024-12-07T21:58:06,770 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for dfaeddf40c94f74d5082b9efaad353be 2024-12-07T21:58:06,771 DEBUG [RS:2;dbf4292a68c8:33333 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;dbf4292a68c8:33333 2024-12-07T21:58:06,771 INFO [RS:2;dbf4292a68c8:33333 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-07T21:58:06,772 INFO [RS:2;dbf4292a68c8:33333 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-07T21:58:06,772 DEBUG [RS:2;dbf4292a68c8:33333 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-07T21:58:06,772 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for dfaeddf40c94f74d5082b9efaad353be 2024-12-07T21:58:06,773 INFO [RS:2;dbf4292a68c8:33333 {}] regionserver.HRegionServer(2659): reportForDuty to master=dbf4292a68c8,42165,1733608683017 with port=33333, startcode=1733608686668 2024-12-07T21:58:06,773 DEBUG [RS:2;dbf4292a68c8:33333 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-07T21:58:06,773 INFO [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened dfaeddf40c94f74d5082b9efaad353be; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62626421, jitterRate=-0.06679360568523407}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T21:58:06,773 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dfaeddf40c94f74d5082b9efaad353be 2024-12-07T21:58:06,774 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for dfaeddf40c94f74d5082b9efaad353be: Running coprocessor pre-open hook at 1733608686751Writing region info on filesystem at 1733608686751Initializing all the Stores at 1733608686751Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733608686751Cleaning up temporary data from old regions at 1733608686770 (+19 ms)Running coprocessor post-open hooks at 1733608686773 (+3 ms)Region opened successfully at 1733608686774 (+1 ms) 2024-12-07T21:58:06,775 INFO [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1733608684459,,1733608684463.dfaeddf40c94f74d5082b9efaad353be., pid=9, masterSystemTime=1733608686747 2024-12-07T21:58:06,775 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59049, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-07T21:58:06,776 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42165 {}] master.ServerManager(363): Checking decommissioned status of RegionServer dbf4292a68c8,33333,1733608686668 2024-12-07T21:58:06,776 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42165 {}] master.ServerManager(517): Registering regionserver=dbf4292a68c8,33333,1733608686668 2024-12-07T21:58:06,779 DEBUG [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1733608684459,,1733608684463.dfaeddf40c94f74d5082b9efaad353be. 2024-12-07T21:58:06,779 INFO [RS_OPEN_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1733608684459,,1733608684463.dfaeddf40c94f74d5082b9efaad353be. 2024-12-07T21:58:06,779 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=dfaeddf40c94f74d5082b9efaad353be, regionState=OPEN, openSeqNum=10, regionLocation=dbf4292a68c8,40365,1733608683183 2024-12-07T21:58:06,780 DEBUG [RS:2;dbf4292a68c8:33333 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442 2024-12-07T21:58:06,780 DEBUG [RS:2;dbf4292a68c8:33333 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-07T21:58:06,780 DEBUG [RS:2;dbf4292a68c8:33333 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-07T21:58:06,783 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure dfaeddf40c94f74d5082b9efaad353be, server=dbf4292a68c8,40365,1733608683183 because future has completed 2024-12-07T21:58:06,786 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-12-07T21:58:06,787 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure dfaeddf40c94f74d5082b9efaad353be, server=dbf4292a68c8,40365,1733608683183 in 192 msec 2024-12-07T21:58:06,789 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server dbf4292a68c8,35277,1733608683217 after splitting done 2024-12-07T21:58:06,790 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-07T21:58:06,790 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733608684459, region=dfaeddf40c94f74d5082b9efaad353be, ASSIGN in 351 msec 2024-12-07T21:58:06,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42165-0x1000256f48f0000, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-07T21:58:06,791 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure dbf4292a68c8,35277,1733608683217, splitWal=true, meta=false in 372 msec 2024-12-07T21:58:06,792 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [dbf4292a68c8,33333,1733608686668] 2024-12-07T21:58:06,792 DEBUG [RS:2;dbf4292a68c8:33333 {}] zookeeper.ZKUtil(111): regionserver:33333-0x1000256f48f0003, quorum=127.0.0.1:65259, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/dbf4292a68c8,33333,1733608686668 2024-12-07T21:58:06,792 WARN [RS:2;dbf4292a68c8:33333 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-07T21:58:06,793 INFO [RS:2;dbf4292a68c8:33333 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T21:58:06,793 DEBUG [RS:2;dbf4292a68c8:33333 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/WALs/dbf4292a68c8,33333,1733608686668 2024-12-07T21:58:06,800 INFO [RS:2;dbf4292a68c8:33333 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-07T21:58:06,805 INFO [RS:2;dbf4292a68c8:33333 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-07T21:58:06,811 INFO [RS:2;dbf4292a68c8:33333 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-07T21:58:06,811 INFO [RS:2;dbf4292a68c8:33333 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T21:58:06,811 INFO [RS:2;dbf4292a68c8:33333 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-07T21:58:06,812 INFO [RS:2;dbf4292a68c8:33333 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-07T21:58:06,812 INFO [RS:2;dbf4292a68c8:33333 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-07T21:58:06,812 DEBUG [RS:2;dbf4292a68c8:33333 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:58:06,812 DEBUG [RS:2;dbf4292a68c8:33333 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:58:06,813 DEBUG [RS:2;dbf4292a68c8:33333 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:58:06,813 DEBUG [RS:2;dbf4292a68c8:33333 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:58:06,813 DEBUG [RS:2;dbf4292a68c8:33333 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:58:06,813 DEBUG [RS:2;dbf4292a68c8:33333 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/dbf4292a68c8:0, corePoolSize=2, maxPoolSize=2 2024-12-07T21:58:06,813 DEBUG [RS:2;dbf4292a68c8:33333 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:58:06,813 DEBUG [RS:2;dbf4292a68c8:33333 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:58:06,813 DEBUG [RS:2;dbf4292a68c8:33333 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:58:06,813 DEBUG [RS:2;dbf4292a68c8:33333 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:58:06,813 DEBUG [RS:2;dbf4292a68c8:33333 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:58:06,813 DEBUG [RS:2;dbf4292a68c8:33333 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/dbf4292a68c8:0, corePoolSize=1, maxPoolSize=1 2024-12-07T21:58:06,813 DEBUG [RS:2;dbf4292a68c8:33333 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/dbf4292a68c8:0, corePoolSize=3, maxPoolSize=3 2024-12-07T21:58:06,813 DEBUG [RS:2;dbf4292a68c8:33333 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/dbf4292a68c8:0, corePoolSize=3, maxPoolSize=3 2024-12-07T21:58:06,819 INFO [RS:2;dbf4292a68c8:33333 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T21:58:06,819 INFO [RS:2;dbf4292a68c8:33333 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T21:58:06,819 INFO [RS:2;dbf4292a68c8:33333 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T21:58:06,819 INFO [RS:2;dbf4292a68c8:33333 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-07T21:58:06,819 INFO [RS:2;dbf4292a68c8:33333 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-07T21:58:06,819 INFO [RS:2;dbf4292a68c8:33333 {}] hbase.ChoreService(168): Chore ScheduledChore name=dbf4292a68c8,33333,1733608686668-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-07T21:58:06,841 INFO [RS:2;dbf4292a68c8:33333 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-07T21:58:06,841 INFO [RS:2;dbf4292a68c8:33333 {}] hbase.ChoreService(168): Chore ScheduledChore name=dbf4292a68c8,33333,1733608686668-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T21:58:06,841 INFO [RS:2;dbf4292a68c8:33333 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T21:58:06,841 INFO [RS:2;dbf4292a68c8:33333 {}] regionserver.Replication(171): dbf4292a68c8,33333,1733608686668 started 2024-12-07T21:58:06,861 INFO [RS:2;dbf4292a68c8:33333 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T21:58:06,861 INFO [RS:2;dbf4292a68c8:33333 {}] regionserver.HRegionServer(1482): Serving as dbf4292a68c8,33333,1733608686668, RpcServer on dbf4292a68c8/172.17.0.2:33333, sessionid=0x1000256f48f0003 2024-12-07T21:58:06,861 DEBUG [RS:2;dbf4292a68c8:33333 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-07T21:58:06,861 DEBUG [RS:2;dbf4292a68c8:33333 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager dbf4292a68c8,33333,1733608686668 2024-12-07T21:58:06,861 DEBUG [RS:2;dbf4292a68c8:33333 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dbf4292a68c8,33333,1733608686668' 2024-12-07T21:58:06,862 DEBUG [RS:2;dbf4292a68c8:33333 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-07T21:58:06,862 DEBUG [RS:2;dbf4292a68c8:33333 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-07T21:58:06,863 DEBUG [RS:2;dbf4292a68c8:33333 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-07T21:58:06,863 DEBUG [RS:2;dbf4292a68c8:33333 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-07T21:58:06,863 DEBUG [RS:2;dbf4292a68c8:33333 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager dbf4292a68c8,33333,1733608686668 2024-12-07T21:58:06,863 DEBUG [RS:2;dbf4292a68c8:33333 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dbf4292a68c8,33333,1733608686668' 2024-12-07T21:58:06,863 DEBUG [RS:2;dbf4292a68c8:33333 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-07T21:58:06,863 DEBUG [RS:2;dbf4292a68c8:33333 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-07T21:58:06,864 DEBUG [RS:2;dbf4292a68c8:33333 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-07T21:58:06,864 INFO [RS:2;dbf4292a68c8:33333 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-07T21:58:06,864 INFO [RS:2;dbf4292a68c8:33333 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-07T21:58:06,965 INFO [RS:2;dbf4292a68c8:33333 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T21:58:06,966 INFO [RS:2;dbf4292a68c8:33333 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=dbf4292a68c8%2C33333%2C1733608686668, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/WALs/dbf4292a68c8,33333,1733608686668, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/oldWALs, maxLogs=55 2024-12-07T21:58:06,986 INFO [RS:2;dbf4292a68c8:33333 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/WALs/dbf4292a68c8,33333,1733608686668/dbf4292a68c8%2C33333%2C1733608686668.1733608686967 2024-12-07T21:58:06,987 DEBUG [RS:2;dbf4292a68c8:33333 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-07T21:58:07,223 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-07T21:58:07,223 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-07T21:58:07,224 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1733608684459 2024-12-07T21:58:07,224 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1733608684459 Metrics about Tables on a single HBase RegionServer 2024-12-07T21:58:07,225 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-07T21:58:07,225 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-07T21:58:07,225 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-07T21:58:07,225 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-07T21:58:07,361 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-07T21:58:07,362 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-07T21:58:07,362 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T21:58:07,362 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T21:58:07,362 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T21:58:07,362 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T21:58:07,362 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-07T21:58:07,362 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1190287876, stopped=false 2024-12-07T21:58:07,362 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=dbf4292a68c8,42165,1733608683017 2024-12-07T21:58:07,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42165-0x1000256f48f0000, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-07T21:58:07,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40365-0x1000256f48f0001, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-07T21:58:07,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42165-0x1000256f48f0000, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T21:58:07,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33333-0x1000256f48f0003, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-07T21:58:07,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40365-0x1000256f48f0001, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T21:58:07,411 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T21:58:07,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33333-0x1000256f48f0003, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T21:58:07,412 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-07T21:58:07,412 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T21:58:07,412 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T21:58:07,412 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'dbf4292a68c8,40365,1733608683183' ***** 2024-12-07T21:58:07,412 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42165-0x1000256f48f0000, quorum=127.0.0.1:65259, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T21:58:07,412 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-07T21:58:07,412 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'dbf4292a68c8,33333,1733608686668' ***** 2024-12-07T21:58:07,412 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-07T21:58:07,413 INFO [RS:0;dbf4292a68c8:40365 {}] bucket.BucketCache(1738): Disabling cache 2024-12-07T21:58:07,413 INFO [RS:2;dbf4292a68c8:33333 {}] bucket.BucketCache(1738): Disabling cache 2024-12-07T21:58:07,413 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-07T21:58:07,413 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-07T21:58:07,413 INFO [RS:2;dbf4292a68c8:33333 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05a8dc90-5822-7fba-c9a9-13ddf09bc5ac/bucket.persistence 2024-12-07T21:58:07,413 INFO [RS:0;dbf4292a68c8:40365 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/05a8dc90-5822-7fba-c9a9-13ddf09bc5ac/bucket.persistence 2024-12-07T21:58:07,413 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-07T21:58:07,413 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-07T21:58:07,414 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-07T21:58:07,414 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-07T21:58:07,414 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40365-0x1000256f48f0001, quorum=127.0.0.1:65259, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T21:58:07,414 INFO [RS:0;dbf4292a68c8:40365 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-07T21:58:07,414 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33333-0x1000256f48f0003, quorum=127.0.0.1:65259, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T21:58:07,414 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-07T21:58:07,414 INFO [RS:2;dbf4292a68c8:33333 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-07T21:58:07,414 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-07T21:58:07,419 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-07T21:58:07,424 DEBUG [RS:0;dbf4292a68c8:40365 {}] bucket.BucketCache(1487): Thread RS:0;dbf4292a68c8:40365 started persisting bucket cache to file 2024-12-07T21:58:07,424 DEBUG [RS:0;dbf4292a68c8:40365 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-07T21:58:07,424 DEBUG [RS:0;dbf4292a68c8:40365 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-07T21:58:07,424 DEBUG [RS:2;dbf4292a68c8:33333 {}] bucket.BucketCache(1487): Thread RS:2;dbf4292a68c8:33333 started persisting bucket cache to file 2024-12-07T21:58:07,424 DEBUG [RS:2;dbf4292a68c8:33333 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-07T21:58:07,424 DEBUG [RS:2;dbf4292a68c8:33333 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-07T21:58:07,427 INFO [regionserver/dbf4292a68c8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T21:58:07,427 DEBUG [RS:2;dbf4292a68c8:33333 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-07T21:58:07,427 DEBUG [RS:0;dbf4292a68c8:40365 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-07T21:58:07,427 DEBUG [RS:2;dbf4292a68c8:33333 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733608687424 2024-12-07T21:58:07,427 DEBUG [RS:0;dbf4292a68c8:40365 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733608687424 2024-12-07T21:58:07,427 DEBUG [RS:0;dbf4292a68c8:40365 {}] bucket.BucketCache(1509): Thread RS:0;dbf4292a68c8:40365 finished persisting bucket cache to file, renaming 2024-12-07T21:58:07,428 INFO [RS:0;dbf4292a68c8:40365 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-07T21:58:07,428 INFO [RS:0;dbf4292a68c8:40365 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-07T21:58:07,428 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-07T21:58:07,428 INFO [RS:0;dbf4292a68c8:40365 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-07T21:58:07,428 INFO [RS:0;dbf4292a68c8:40365 {}] regionserver.HRegionServer(3091): Received CLOSE for dfaeddf40c94f74d5082b9efaad353be 2024-12-07T21:58:07,428 INFO [RS:0;dbf4292a68c8:40365 {}] regionserver.HRegionServer(959): stopping server dbf4292a68c8,40365,1733608683183 2024-12-07T21:58:07,428 INFO [RS:0;dbf4292a68c8:40365 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-07T21:58:07,428 INFO [RS:0;dbf4292a68c8:40365 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;dbf4292a68c8:40365. 2024-12-07T21:58:07,428 INFO [RS_CLOSE_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1733608684459,,1733608684463.dfaeddf40c94f74d5082b9efaad353be. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-07T21:58:07,428 DEBUG [RS:0;dbf4292a68c8:40365 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T21:58:07,429 DEBUG [RS:0;dbf4292a68c8:40365 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T21:58:07,429 DEBUG [RS_CLOSE_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing dfaeddf40c94f74d5082b9efaad353be, disabling compactions & flushes 2024-12-07T21:58:07,429 INFO [RS:0;dbf4292a68c8:40365 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-07T21:58:07,429 INFO [RS_CLOSE_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733608684459,,1733608684463.dfaeddf40c94f74d5082b9efaad353be. 2024-12-07T21:58:07,429 DEBUG [RS_CLOSE_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733608684459,,1733608684463.dfaeddf40c94f74d5082b9efaad353be. 2024-12-07T21:58:07,429 DEBUG [RS_CLOSE_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733608684459,,1733608684463.dfaeddf40c94f74d5082b9efaad353be. after waiting 0 ms 2024-12-07T21:58:07,429 DEBUG [RS_CLOSE_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733608684459,,1733608684463.dfaeddf40c94f74d5082b9efaad353be. 2024-12-07T21:58:07,429 INFO [RS:0;dbf4292a68c8:40365 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-07T21:58:07,429 INFO [RS:0;dbf4292a68c8:40365 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-07T21:58:07,429 INFO [RS:0;dbf4292a68c8:40365 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-07T21:58:07,430 INFO [RS:0;dbf4292a68c8:40365 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-07T21:58:07,430 DEBUG [RS:0;dbf4292a68c8:40365 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, dfaeddf40c94f74d5082b9efaad353be=testBlockEvictionOnGracefulStop1733608684459,,1733608684463.dfaeddf40c94f74d5082b9efaad353be.} 2024-12-07T21:58:07,430 DEBUG [RS:0;dbf4292a68c8:40365 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, dfaeddf40c94f74d5082b9efaad353be 2024-12-07T21:58:07,430 INFO [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-07T21:58:07,430 DEBUG [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-07T21:58:07,430 INFO [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-07T21:58:07,430 DEBUG [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-07T21:58:07,431 DEBUG [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-07T21:58:07,431 DEBUG [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-07T21:58:07,431 DEBUG [StoreFileCloser-dfaeddf40c94f74d5082b9efaad353be-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/default/testBlockEvictionOnGracefulStop1733608684459/dfaeddf40c94f74d5082b9efaad353be/family/64ca6b6064e54c399853fdae078cb99d 2024-12-07T21:58:07,431 INFO [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.96 KB heapSize=5.58 KB 2024-12-07T21:58:07,435 DEBUG [RS_CLOSE_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/default/testBlockEvictionOnGracefulStop1733608684459/dfaeddf40c94f74d5082b9efaad353be/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-12-07T21:58:07,436 INFO [RS_CLOSE_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733608684459,,1733608684463.dfaeddf40c94f74d5082b9efaad353be. 2024-12-07T21:58:07,436 DEBUG [RS_CLOSE_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for dfaeddf40c94f74d5082b9efaad353be: Waiting for close lock at 1733608687428Running coprocessor pre-close hooks at 1733608687428Disabling compacts and flushes for region at 1733608687428Disabling writes for close at 1733608687429 (+1 ms)Writing region close event to WAL at 1733608687431 (+2 ms)Running coprocessor post-close hooks at 1733608687435 (+4 ms)Closed at 1733608687436 (+1 ms) 2024-12-07T21:58:07,436 DEBUG [RS_CLOSE_REGION-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1733608684459,,1733608684463.dfaeddf40c94f74d5082b9efaad353be. 2024-12-07T21:58:07,440 DEBUG [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 2d62f9bcc0ae4f3fa9ddaa10b58a5104 2024-12-07T21:58:07,443 DEBUG [RS:2;dbf4292a68c8:33333 {}] bucket.BucketCache(1509): Thread RS:2;dbf4292a68c8:33333 finished persisting bucket cache to file, renaming 2024-12-07T21:58:07,443 WARN [RS:2;dbf4292a68c8:33333 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-12-07T21:58:07,443 INFO [RS:2;dbf4292a68c8:33333 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-07T21:58:07,444 INFO [RS:2;dbf4292a68c8:33333 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-07T21:58:07,444 INFO [RS:2;dbf4292a68c8:33333 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-07T21:58:07,444 INFO [RS:2;dbf4292a68c8:33333 {}] regionserver.HRegionServer(959): stopping server dbf4292a68c8,33333,1733608686668 2024-12-07T21:58:07,444 INFO [RS:2;dbf4292a68c8:33333 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-07T21:58:07,444 INFO [RS:2;dbf4292a68c8:33333 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;dbf4292a68c8:33333. 2024-12-07T21:58:07,444 DEBUG [RS:2;dbf4292a68c8:33333 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T21:58:07,444 DEBUG [RS:2;dbf4292a68c8:33333 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T21:58:07,444 INFO [RS:2;dbf4292a68c8:33333 {}] regionserver.HRegionServer(976): stopping server dbf4292a68c8,33333,1733608686668; all regions closed. 2024-12-07T21:58:07,447 DEBUG [RS:2;dbf4292a68c8:33333 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/oldWALs 2024-12-07T21:58:07,447 INFO [RS:2;dbf4292a68c8:33333 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL dbf4292a68c8%2C33333%2C1733608686668:(num 1733608686967) 2024-12-07T21:58:07,447 DEBUG [RS:2;dbf4292a68c8:33333 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T21:58:07,447 INFO [RS:2;dbf4292a68c8:33333 {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T21:58:07,447 INFO [RS:2;dbf4292a68c8:33333 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-07T21:58:07,448 INFO [RS:2;dbf4292a68c8:33333 {}] hbase.ChoreService(370): Chore service for: regionserver/dbf4292a68c8:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-07T21:58:07,448 INFO [RS:2;dbf4292a68c8:33333 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-07T21:58:07,448 INFO [regionserver/dbf4292a68c8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-07T21:58:07,448 INFO [RS:2;dbf4292a68c8:33333 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-07T21:58:07,448 INFO [RS:2;dbf4292a68c8:33333 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-07T21:58:07,448 INFO [RS:2;dbf4292a68c8:33333 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-07T21:58:07,448 INFO [RS:2;dbf4292a68c8:33333 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33333 2024-12-07T21:58:07,459 DEBUG [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740/.tmp/info/2d62f9bcc0ae4f3fa9ddaa10b58a5104 is 209, key is testBlockEvictionOnGracefulStop1733608684459,,1733608684463.dfaeddf40c94f74d5082b9efaad353be./info:regioninfo/1733608686779/Put/seqid=0 2024-12-07T21:58:07,460 INFO [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.74 KB at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740/.tmp/info/2d62f9bcc0ae4f3fa9ddaa10b58a5104 2024-12-07T21:58:07,467 INFO [regionserver/dbf4292a68c8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T21:58:07,468 DEBUG [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f437e097607246988fb8bb8df968ed7d 2024-12-07T21:58:07,474 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33333-0x1000256f48f0003, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/dbf4292a68c8,33333,1733608686668 2024-12-07T21:58:07,474 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42165-0x1000256f48f0000, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-07T21:58:07,474 INFO [RS:2;dbf4292a68c8:33333 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-07T21:58:07,474 ERROR [Time-limited test-EventThread {}] zookeeper.ClientCnxn$EventThread(581): Error while calling watcher. java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.trace.TraceUtil$$Lambda$249/0x00007f74d847cd88@636cfa75 rejected from java.util.concurrent.ThreadPoolExecutor@266e687c[Shutting down, pool size = 1, active threads = 0, queued tasks = 0, completed tasks = 3] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1360) ~[?:?] at java.util.concurrent.Executors$DelegatedExecutorService.execute(Executors.java:721) ~[?:?] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.process(ZKWatcher.java:613) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:579) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:554) ~[zookeeper-3.8.4.jar:3.8.4] 2024-12-07T21:58:07,482 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [dbf4292a68c8,33333,1733608686668] 2024-12-07T21:58:07,485 DEBUG [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740/.tmp/ns/f437e097607246988fb8bb8df968ed7d is 43, key is default/ns:d/1733608684358/Put/seqid=0 2024-12-07T21:58:07,486 INFO [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740/.tmp/ns/f437e097607246988fb8bb8df968ed7d 2024-12-07T21:58:07,491 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/dbf4292a68c8,33333,1733608686668 already deleted, retry=false 2024-12-07T21:58:07,491 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; dbf4292a68c8,33333,1733608686668 expired; onlineServers=1 2024-12-07T21:58:07,494 DEBUG [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c0f1b09d47db48758aa0b045f3063c9a 2024-12-07T21:58:07,511 DEBUG [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740/.tmp/table/c0f1b09d47db48758aa0b045f3063c9a is 80, key is testBlockEvictionOnGracefulStop1733608684459/table:state/1733608684849/Put/seqid=0 2024-12-07T21:58:07,512 INFO [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740/.tmp/table/c0f1b09d47db48758aa0b045f3063c9a 2024-12-07T21:58:07,515 DEBUG [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740/.tmp/info/2d62f9bcc0ae4f3fa9ddaa10b58a5104 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740/info/2d62f9bcc0ae4f3fa9ddaa10b58a5104 2024-12-07T21:58:07,517 DEBUG [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740/info/2d62f9bcc0ae4f3fa9ddaa10b58a5104, delay=921 ms 2024-12-07T21:58:07,518 INFO [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740/info/2d62f9bcc0ae4f3fa9ddaa10b58a5104, entries=14, sequenceid=13, filesize=7.5 K 2024-12-07T21:58:07,519 DEBUG [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740/.tmp/ns/f437e097607246988fb8bb8df968ed7d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740/ns/f437e097607246988fb8bb8df968ed7d 2024-12-07T21:58:07,520 DEBUG [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740/ns/f437e097607246988fb8bb8df968ed7d, delay=989 ms 2024-12-07T21:58:07,521 INFO [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740/ns/f437e097607246988fb8bb8df968ed7d, entries=2, sequenceid=13, filesize=5.0 K 2024-12-07T21:58:07,522 DEBUG [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740/.tmp/table/c0f1b09d47db48758aa0b045f3063c9a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740/table/c0f1b09d47db48758aa0b045f3063c9a 2024-12-07T21:58:07,524 DEBUG [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740/table/c0f1b09d47db48758aa0b045f3063c9a, delay=977 ms 2024-12-07T21:58:07,525 INFO [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740/table/c0f1b09d47db48758aa0b045f3063c9a, entries=2, sequenceid=13, filesize=5.3 K 2024-12-07T21:58:07,526 INFO [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 95ms, sequenceid=13, compaction requested=false 2024-12-07T21:58:07,527 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740/info/2d62f9bcc0ae4f3fa9ddaa10b58a5104 2024-12-07T21:58:07,527 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740/ns/f437e097607246988fb8bb8df968ed7d 2024-12-07T21:58:07,529 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740/table/c0f1b09d47db48758aa0b045f3063c9a 2024-12-07T21:58:07,534 DEBUG [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/data/hbase/meta/1588230740/recovered.edits/16.seqid, newMaxSeqId=16, maxSeqId=1 2024-12-07T21:58:07,534 DEBUG [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-07T21:58:07,534 INFO [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-07T21:58:07,535 DEBUG [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733608687430Running coprocessor pre-close hooks at 1733608687430Disabling compacts and flushes for region at 1733608687430Disabling writes for close at 1733608687431 (+1 ms)Obtaining lock to block concurrent updates at 1733608687431Preparing flush snapshotting stores in 1588230740 at 1733608687431Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3035, getHeapSize=5648, getOffHeapSize=0, getCellsCount=22 at 1733608687431Flushing stores of hbase:meta,,1.1588230740 at 1733608687432 (+1 ms)Flushing 1588230740/info: creating writer at 1733608687432Flushing 1588230740/info: appending metadata at 1733608687458 (+26 ms)Flushing 1588230740/info: closing flushed file at 1733608687459 (+1 ms)Flushing 1588230740/ns: creating writer at 1733608687462 (+3 ms)Flushing 1588230740/ns: appending metadata at 1733608687485 (+23 ms)Flushing 1588230740/ns: closing flushed file at 1733608687485Flushing 1588230740/table: creating writer at 1733608687489 (+4 ms)Flushing 1588230740/table: appending metadata at 1733608687510 (+21 ms)Flushing 1588230740/table: closing flushed file at 1733608687511 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@372ab08c: reopening flushed file at 1733608687515 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@28afd49: reopening flushed file at 1733608687518 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b0992b7: reopening flushed file at 1733608687522 (+4 ms)Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 95ms, sequenceid=13, compaction requested=false at 1733608687526 (+4 ms)Writing region close event to WAL at 1733608687529 (+3 ms)Running coprocessor post-close hooks at 1733608687534 (+5 ms)Closed at 1733608687534 2024-12-07T21:58:07,535 DEBUG [RS_CLOSE_META-regionserver/dbf4292a68c8:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-07T21:58:07,583 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33333-0x1000256f48f0003, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T21:58:07,583 INFO [RS:2;dbf4292a68c8:33333 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-07T21:58:07,583 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33333-0x1000256f48f0003, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T21:58:07,583 INFO [RS:2;dbf4292a68c8:33333 {}] regionserver.HRegionServer(1031): Exiting; stopping=dbf4292a68c8,33333,1733608686668; zookeeper connection closed. 2024-12-07T21:58:07,583 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2d9023fa {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2d9023fa 2024-12-07T21:58:07,630 INFO [RS:0;dbf4292a68c8:40365 {}] regionserver.HRegionServer(976): stopping server dbf4292a68c8,40365,1733608683183; all regions closed. 2024-12-07T21:58:07,632 DEBUG [RS:0;dbf4292a68c8:40365 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/oldWALs 2024-12-07T21:58:07,632 INFO [RS:0;dbf4292a68c8:40365 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL dbf4292a68c8%2C40365%2C1733608683183.meta:.meta(num 1733608684300) 2024-12-07T21:58:07,634 DEBUG [RS:0;dbf4292a68c8:40365 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/oldWALs 2024-12-07T21:58:07,634 INFO [RS:0;dbf4292a68c8:40365 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL dbf4292a68c8%2C40365%2C1733608683183:(num 1733608683901) 2024-12-07T21:58:07,634 DEBUG [RS:0;dbf4292a68c8:40365 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T21:58:07,634 INFO [RS:0;dbf4292a68c8:40365 {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T21:58:07,634 INFO [RS:0;dbf4292a68c8:40365 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-07T21:58:07,634 INFO [RS:0;dbf4292a68c8:40365 {}] hbase.ChoreService(370): Chore service for: regionserver/dbf4292a68c8:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-07T21:58:07,634 INFO [RS:0;dbf4292a68c8:40365 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-07T21:58:07,635 INFO [RS:0;dbf4292a68c8:40365 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40365 2024-12-07T21:58:07,636 INFO [regionserver/dbf4292a68c8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-07T21:58:07,669 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42165-0x1000256f48f0000, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-07T21:58:07,669 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40365-0x1000256f48f0001, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/dbf4292a68c8,40365,1733608683183 2024-12-07T21:58:07,670 INFO [RS:0;dbf4292a68c8:40365 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-07T21:58:07,670 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [dbf4292a68c8,40365,1733608683183] 2024-12-07T21:58:07,686 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/dbf4292a68c8,40365,1733608683183 already deleted, retry=false 2024-12-07T21:58:07,686 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; dbf4292a68c8,40365,1733608683183 expired; onlineServers=0 2024-12-07T21:58:07,686 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'dbf4292a68c8,42165,1733608683017' ***** 2024-12-07T21:58:07,686 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-07T21:58:07,686 INFO [M:0;dbf4292a68c8:42165 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-07T21:58:07,686 INFO [M:0;dbf4292a68c8:42165 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-07T21:58:07,686 DEBUG [M:0;dbf4292a68c8:42165 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-07T21:58:07,687 DEBUG [M:0;dbf4292a68c8:42165 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-07T21:58:07,687 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-07T21:58:07,687 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster-HFileCleaner.small.0-1733608683587 {}] cleaner.HFileCleaner(306): Exit Thread[master/dbf4292a68c8:0:becomeActiveMaster-HFileCleaner.small.0-1733608683587,5,FailOnTimeoutGroup] 2024-12-07T21:58:07,687 DEBUG [master/dbf4292a68c8:0:becomeActiveMaster-HFileCleaner.large.0-1733608683587 {}] cleaner.HFileCleaner(306): Exit Thread[master/dbf4292a68c8:0:becomeActiveMaster-HFileCleaner.large.0-1733608683587,5,FailOnTimeoutGroup] 2024-12-07T21:58:07,687 INFO [M:0;dbf4292a68c8:42165 {}] hbase.ChoreService(370): Chore service for: master/dbf4292a68c8:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-07T21:58:07,688 INFO [M:0;dbf4292a68c8:42165 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-07T21:58:07,688 DEBUG [M:0;dbf4292a68c8:42165 {}] master.HMaster(1795): Stopping service threads 2024-12-07T21:58:07,688 INFO [M:0;dbf4292a68c8:42165 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-07T21:58:07,688 INFO [M:0;dbf4292a68c8:42165 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T21:58:07,688 INFO [M:0;dbf4292a68c8:42165 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T21:58:07,688 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-07T21:58:07,694 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42165-0x1000256f48f0000, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-07T21:58:07,694 DEBUG [M:0;dbf4292a68c8:42165 {}] zookeeper.ZKUtil(347): master:42165-0x1000256f48f0000, quorum=127.0.0.1:65259, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-07T21:58:07,695 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42165-0x1000256f48f0000, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T21:58:07,695 WARN [M:0;dbf4292a68c8:42165 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-07T21:58:07,695 INFO [M:0;dbf4292a68c8:42165 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/.lastflushedseqids 2024-12-07T21:58:07,702 INFO [M:0;dbf4292a68c8:42165 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T21:58:07,702 INFO [M:0;dbf4292a68c8:42165 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-07T21:58:07,702 DEBUG [M:0;dbf4292a68c8:42165 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-07T21:58:07,702 INFO [M:0;dbf4292a68c8:42165 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T21:58:07,702 DEBUG [M:0;dbf4292a68c8:42165 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T21:58:07,703 DEBUG [M:0;dbf4292a68c8:42165 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-07T21:58:07,703 DEBUG [M:0;dbf4292a68c8:42165 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T21:58:07,703 INFO [M:0;dbf4292a68c8:42165 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=32.84 KB heapSize=41.09 KB 2024-12-07T21:58:07,712 DEBUG [M:0;dbf4292a68c8:42165 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a31eda48d9684c2da0e6df2bf597e775 2024-12-07T21:58:07,734 DEBUG [M:0;dbf4292a68c8:42165 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a31eda48d9684c2da0e6df2bf597e775 is 82, key is hbase:meta,,1/info:regioninfo/1733608684329/Put/seqid=0 2024-12-07T21:58:07,734 INFO [M:0;dbf4292a68c8:42165 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a31eda48d9684c2da0e6df2bf597e775 2024-12-07T21:58:07,744 DEBUG [M:0;dbf4292a68c8:42165 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ad029ea0b0054e7ab44b522b902e6b3b 2024-12-07T21:58:07,759 DEBUG [M:0;dbf4292a68c8:42165 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ad029ea0b0054e7ab44b522b902e6b3b is 783, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733608684853/Put/seqid=0 2024-12-07T21:58:07,760 INFO [M:0;dbf4292a68c8:42165 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.10 KB at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ad029ea0b0054e7ab44b522b902e6b3b 2024-12-07T21:58:07,770 DEBUG [M:0;dbf4292a68c8:42165 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 9800b5252377425ba1d5079f6afd3fc4 2024-12-07T21:58:07,778 INFO [RS:0;dbf4292a68c8:40365 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-07T21:58:07,778 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40365-0x1000256f48f0001, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T21:58:07,778 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40365-0x1000256f48f0001, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T21:58:07,778 INFO [RS:0;dbf4292a68c8:40365 {}] regionserver.HRegionServer(1031): Exiting; stopping=dbf4292a68c8,40365,1733608683183; zookeeper connection closed. 2024-12-07T21:58:07,779 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@212c635c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@212c635c 2024-12-07T21:58:07,779 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-07T21:58:07,786 DEBUG [M:0;dbf4292a68c8:42165 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9800b5252377425ba1d5079f6afd3fc4 is 69, key is dbf4292a68c8,33333,1733608686668/rs:state/1733608686776/Put/seqid=0 2024-12-07T21:58:07,786 INFO [M:0;dbf4292a68c8:42165 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9800b5252377425ba1d5079f6afd3fc4 2024-12-07T21:58:07,788 INFO [M:0;dbf4292a68c8:42165 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9800b5252377425ba1d5079f6afd3fc4 2024-12-07T21:58:07,789 DEBUG [M:0;dbf4292a68c8:42165 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a31eda48d9684c2da0e6df2bf597e775 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a31eda48d9684c2da0e6df2bf597e775 2024-12-07T21:58:07,791 INFO [M:0;dbf4292a68c8:42165 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a31eda48d9684c2da0e6df2bf597e775, entries=8, sequenceid=83, filesize=5.5 K 2024-12-07T21:58:07,792 DEBUG [M:0;dbf4292a68c8:42165 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ad029ea0b0054e7ab44b522b902e6b3b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ad029ea0b0054e7ab44b522b902e6b3b 2024-12-07T21:58:07,794 INFO [M:0;dbf4292a68c8:42165 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ad029ea0b0054e7ab44b522b902e6b3b, entries=9, sequenceid=83, filesize=6.4 K 2024-12-07T21:58:07,794 DEBUG [M:0;dbf4292a68c8:42165 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9800b5252377425ba1d5079f6afd3fc4 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9800b5252377425ba1d5079f6afd3fc4 2024-12-07T21:58:07,796 INFO [M:0;dbf4292a68c8:42165 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9800b5252377425ba1d5079f6afd3fc4 2024-12-07T21:58:07,796 INFO [M:0;dbf4292a68c8:42165 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/da892f72-9cd3-26b3-2e52-98f9ff84ed38/ac4a857a-793f-51ee-d125-dcf65373d442/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9800b5252377425ba1d5079f6afd3fc4, entries=3, sequenceid=83, filesize=5.3 K 2024-12-07T21:58:07,797 INFO [M:0;dbf4292a68c8:42165 {}] regionserver.HRegion(3140): Finished flush of dataSize ~32.84 KB/33626, heapSize ~40.79 KB/41768, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 94ms, sequenceid=83, compaction requested=false 2024-12-07T21:58:07,799 INFO [M:0;dbf4292a68c8:42165 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T21:58:07,799 DEBUG [M:0;dbf4292a68c8:42165 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733608687702Disabling compacts and flushes for region at 1733608687702Disabling writes for close at 1733608687703 (+1 ms)Obtaining lock to block concurrent updates at 1733608687703Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733608687703Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=33626, getHeapSize=42008, getOffHeapSize=0, getCellsCount=98 at 1733608687703Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733608687704 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733608687704Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733608687733 (+29 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733608687733Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733608687737 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733608687759 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733608687759Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733608687762 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733608687785 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733608687785Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@131933b: reopening flushed file at 1733608687789 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@767576b9: reopening flushed file at 1733608687792 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@247ee32f: reopening flushed file at 1733608687794 (+2 ms)Finished flush of dataSize ~32.84 KB/33626, heapSize ~40.79 KB/41768, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 94ms, sequenceid=83, compaction requested=false at 1733608687797 (+3 ms)Writing region close event to WAL at 1733608687798 (+1 ms)Closed at 1733608687798 2024-12-07T21:58:07,801 INFO [M:0;dbf4292a68c8:42165 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-07T21:58:07,801 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-07T21:58:07,801 INFO [M:0;dbf4292a68c8:42165 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42165 2024-12-07T21:58:07,801 INFO [M:0;dbf4292a68c8:42165 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-07T21:58:07,924 INFO [M:0;dbf4292a68c8:42165 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-07T21:58:07,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42165-0x1000256f48f0000, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T21:58:07,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42165-0x1000256f48f0000, quorum=127.0.0.1:65259, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T21:58:07,926 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-07T21:58:07,927 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-07T21:58:07,927 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T21:58:07,933 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=73 (was 51) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=535 (was 477) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=17993 (was 18061)