2024-12-05 12:01:06,345 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-12-05 12:01:06,383 main DEBUG Took 0.035931 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-05 12:01:06,383 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-05 12:01:06,384 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-05 12:01:06,385 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-05 12:01:06,387 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 12:01:06,397 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-05 12:01:06,414 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 12:01:06,417 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 12:01:06,418 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 12:01:06,419 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 12:01:06,420 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 12:01:06,420 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 12:01:06,424 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 12:01:06,424 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 12:01:06,425 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 12:01:06,425 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 12:01:06,427 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 12:01:06,427 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 12:01:06,428 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 12:01:06,428 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 12:01:06,429 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 12:01:06,429 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 12:01:06,430 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 12:01:06,430 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 12:01:06,431 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 12:01:06,431 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 12:01:06,432 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 12:01:06,432 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 12:01:06,437 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 12:01:06,437 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 12:01:06,439 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 12:01:06,439 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-05 12:01:06,441 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 12:01:06,443 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-05 12:01:06,446 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-05 12:01:06,447 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-05 12:01:06,449 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-05 12:01:06,449 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-05 12:01:06,462 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-05 12:01:06,466 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-05 12:01:06,468 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-05 12:01:06,469 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-05 12:01:06,470 main DEBUG createAppenders(={Console}) 2024-12-05 12:01:06,471 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-12-05 12:01:06,471 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-12-05 12:01:06,471 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-12-05 12:01:06,472 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-05 12:01:06,473 main DEBUG OutputStream closed 2024-12-05 12:01:06,473 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-05 12:01:06,473 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-05 12:01:06,474 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-12-05 12:01:06,562 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-05 12:01:06,565 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-05 12:01:06,566 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-05 12:01:06,567 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-05 12:01:06,568 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-05 12:01:06,568 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-05 12:01:06,569 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-05 12:01:06,569 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-05 12:01:06,570 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-05 12:01:06,570 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-05 12:01:06,570 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-05 12:01:06,571 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-05 12:01:06,571 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-05 12:01:06,572 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-05 12:01:06,572 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-05 12:01:06,572 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-05 12:01:06,573 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-05 12:01:06,574 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-05 12:01:06,577 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-05 12:01:06,577 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-12-05 12:01:06,577 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-05 12:01:06,578 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-12-05T12:01:06,594 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-12-05 12:01:06,597 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-05 12:01:06,598 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-05T12:01:06,936 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506 2024-12-05T12:01:06,968 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=1052, ProcessCount=11, AvailableMemoryMB=7747 2024-12-05T12:01:07,102 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-05T12:01:07,246 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T12:01:07,280 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/cluster_d723c6ab-e1d8-a458-6cbb-4af1b3c638db, deleteOnExit=true 2024-12-05T12:01:07,443 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/cluster_d723c6ab-e1d8-a458-6cbb-4af1b3c638db/zookeeper_0, clientPort=51837, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/cluster_d723c6ab-e1d8-a458-6cbb-4af1b3c638db/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/cluster_d723c6ab-e1d8-a458-6cbb-4af1b3c638db/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-05T12:01:07,457 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51837 2024-12-05T12:01:07,458 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T12:01:07,574 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6 with version=8 2024-12-05T12:01:07,575 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/hbase-staging 2024-12-05T12:01:07,676 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-05T12:01:07,950 INFO [Time-limited test {}] client.ConnectionUtils(128): master/80666a11a09f:0 server-side Connection retries=45 2024-12-05T12:01:07,964 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T12:01:07,966 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T12:01:07,976 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T12:01:07,976 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T12:01:07,977 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T12:01:08,172 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-05T12:01:08,284 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T12:01:08,338 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-05T12:01:08,349 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-05T12:01:08,353 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T12:01:08,401 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 118005 (auto-detected) 2024-12-05T12:01:08,402 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-05T12:01:08,426 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41433 2024-12-05T12:01:08,450 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41433 connecting to ZooKeeper ensemble=127.0.0.1:51837 2024-12-05T12:01:08,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:414330x0, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T12:01:08,564 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41433-0x101a6a79fd00000 connected 2024-12-05T12:01:08,650 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T12:01:08,651 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T12:01:08,660 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41433-0x101a6a79fd00000, quorum=127.0.0.1:51837, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T12:01:08,664 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6, hbase.cluster.distributed=false 2024-12-05T12:01:08,692 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41433-0x101a6a79fd00000, quorum=127.0.0.1:51837, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T12:01:08,700 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41433 2024-12-05T12:01:08,700 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41433 2024-12-05T12:01:08,701 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41433 2024-12-05T12:01:08,701 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41433 2024-12-05T12:01:08,702 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41433 2024-12-05T12:01:08,804 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/80666a11a09f:0 server-side Connection retries=45 2024-12-05T12:01:08,806 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T12:01:08,807 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T12:01:08,807 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T12:01:08,807 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T12:01:08,807 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T12:01:08,811 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-05T12:01:08,814 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T12:01:08,815 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34899 2024-12-05T12:01:08,818 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34899 connecting to ZooKeeper ensemble=127.0.0.1:51837 2024-12-05T12:01:08,818 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T12:01:08,819 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T12:01:08,830 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:348990x0, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T12:01:08,831 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:348990x0, quorum=127.0.0.1:51837, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T12:01:08,831 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34899-0x101a6a79fd00001 connected 2024-12-05T12:01:08,838 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-05T12:01:08,853 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/bucket.cache 2024-12-05T12:01:08,860 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-05T12:01:08,861 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-05T12:01:08,863 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-05T12:01:08,866 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-05T12:01:08,866 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-05T12:01:08,874 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-05T12:01:08,876 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-05T12:01:08,877 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/bucket.persistence is complete. 2024-12-05T12:01:08,890 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-05T12:01:08,893 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34899-0x101a6a79fd00001, quorum=127.0.0.1:51837, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-05T12:01:08,899 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34899-0x101a6a79fd00001, quorum=127.0.0.1:51837, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T12:01:08,915 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34899 2024-12-05T12:01:08,919 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34899 2024-12-05T12:01:08,927 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34899 2024-12-05T12:01:08,939 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34899 2024-12-05T12:01:08,939 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34899 2024-12-05T12:01:08,968 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/80666a11a09f:0 server-side Connection retries=45 2024-12-05T12:01:08,968 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T12:01:08,968 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T12:01:08,969 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T12:01:08,969 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T12:01:08,969 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T12:01:08,970 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-05T12:01:08,970 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T12:01:08,971 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33013 2024-12-05T12:01:08,974 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33013 connecting to ZooKeeper ensemble=127.0.0.1:51837 2024-12-05T12:01:08,975 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T12:01:08,976 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T12:01:08,998 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:330130x0, quorum=127.0.0.1:51837, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T12:01:08,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:330130x0, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T12:01:08,999 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-05T12:01:09,001 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33013-0x101a6a79fd00002 connected 2024-12-05T12:01:09,019 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/bucket.cache 2024-12-05T12:01:09,019 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-05T12:01:09,020 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-05T12:01:09,020 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-05T12:01:09,022 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-05T12:01:09,022 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-05T12:01:09,023 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-05T12:01:09,023 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/bucket.persistence is complete. 2024-12-05T12:01:09,031 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-05T12:01:09,038 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-05T12:01:09,040 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33013-0x101a6a79fd00002, quorum=127.0.0.1:51837, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-05T12:01:09,044 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33013-0x101a6a79fd00002, quorum=127.0.0.1:51837, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T12:01:09,063 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33013 2024-12-05T12:01:09,079 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33013 2024-12-05T12:01:09,103 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33013 2024-12-05T12:01:09,115 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33013 2024-12-05T12:01:09,115 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33013 2024-12-05T12:01:09,144 DEBUG [M:0;80666a11a09f:41433 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;80666a11a09f:41433 2024-12-05T12:01:09,163 INFO [master/80666a11a09f:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/80666a11a09f,41433,1733400067757 2024-12-05T12:01:09,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33013-0x101a6a79fd00002, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T12:01:09,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34899-0x101a6a79fd00001, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T12:01:09,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41433-0x101a6a79fd00000, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T12:01:09,186 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41433-0x101a6a79fd00000, quorum=127.0.0.1:51837, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/80666a11a09f,41433,1733400067757 2024-12-05T12:01:09,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34899-0x101a6a79fd00001, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-05T12:01:09,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34899-0x101a6a79fd00001, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:01:09,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41433-0x101a6a79fd00000, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:01:09,223 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33013-0x101a6a79fd00002, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-05T12:01:09,223 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33013-0x101a6a79fd00002, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:01:09,225 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41433-0x101a6a79fd00000, quorum=127.0.0.1:51837, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-05T12:01:09,228 INFO [master/80666a11a09f:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/80666a11a09f,41433,1733400067757 from backup master directory 2024-12-05T12:01:09,238 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34899-0x101a6a79fd00001, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T12:01:09,238 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41433-0x101a6a79fd00000, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/80666a11a09f,41433,1733400067757 2024-12-05T12:01:09,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33013-0x101a6a79fd00002, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T12:01:09,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41433-0x101a6a79fd00000, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T12:01:09,241 WARN [master/80666a11a09f:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T12:01:09,241 INFO [master/80666a11a09f:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=80666a11a09f,41433,1733400067757 2024-12-05T12:01:09,244 INFO [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-05T12:01:09,246 INFO [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-05T12:01:09,257 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/hbase.id] with ID: 8c4ff262-9bf1-4aef-8074-a07876e58646 2024-12-05T12:01:09,258 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/.tmp/hbase.id 2024-12-05T12:01:09,266 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/hbase.id] 2024-12-05T12:01:09,326 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T12:01:09,333 INFO [master/80666a11a09f:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-05T12:01:09,356 INFO [master/80666a11a09f:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 21ms. 2024-12-05T12:01:09,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34899-0x101a6a79fd00001, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:01:09,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33013-0x101a6a79fd00002, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:01:09,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41433-0x101a6a79fd00000, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:01:09,399 INFO [master/80666a11a09f:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T12:01:09,403 INFO [master/80666a11a09f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-05T12:01:09,448 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:01:09,454 INFO [master/80666a11a09f:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T12:01:09,528 INFO [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/MasterData/data/master/store 2024-12-05T12:01:09,586 INFO [master/80666a11a09f:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-05T12:01:09,591 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T12:01:09,594 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-05T12:01:09,594 INFO [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T12:01:09,595 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T12:01:09,598 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-05T12:01:09,598 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T12:01:09,599 INFO [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T12:01:09,599 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733400069594Disabling compacts and flushes for region at 1733400069594Disabling writes for close at 1733400069598 (+4 ms)Writing region close event to WAL at 1733400069599 (+1 ms)Closed at 1733400069599 2024-12-05T12:01:09,600 WARN [master/80666a11a09f:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/MasterData/data/master/store/.initializing 2024-12-05T12:01:09,600 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/MasterData/WALs/80666a11a09f,41433,1733400067757 2024-12-05T12:01:09,618 INFO [master/80666a11a09f:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T12:01:09,638 INFO [master/80666a11a09f:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=80666a11a09f%2C41433%2C1733400067757, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/MasterData/WALs/80666a11a09f,41433,1733400067757, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/MasterData/oldWALs, maxLogs=10 2024-12-05T12:01:09,683 INFO [master/80666a11a09f:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/MasterData/WALs/80666a11a09f,41433,1733400067757/80666a11a09f%2C41433%2C1733400067757.1733400069644 2024-12-05T12:01:09,685 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-05T12:01:09,685 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-05T12:01:09,686 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T12:01:09,691 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T12:01:09,692 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T12:01:09,747 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T12:01:09,808 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-05T12:01:09,814 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:01:09,816 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T12:01:09,818 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T12:01:09,834 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-05T12:01:09,834 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:01:09,835 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T12:01:09,838 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T12:01:09,846 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-05T12:01:09,848 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:01:09,849 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T12:01:09,851 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T12:01:09,861 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-05T12:01:09,861 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:01:09,862 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T12:01:09,862 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T12:01:09,868 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-05T12:01:09,869 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-05T12:01:09,874 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T12:01:09,875 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T12:01:09,885 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T12:01:09,890 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T12:01:09,911 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T12:01:09,913 INFO [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61604990, jitterRate=-0.08201411366462708}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T12:01:09,944 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733400069709Initializing all the Stores at 1733400069710 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733400069711 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733400069715 (+4 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733400069716 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733400069716Cleaning up temporary data from old regions at 1733400069880 (+164 ms)Region opened successfully at 1733400069944 (+64 ms) 2024-12-05T12:01:09,955 INFO [master/80666a11a09f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-05T12:01:10,030 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@326f38b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=80666a11a09f/172.17.0.2:0 2024-12-05T12:01:10,081 INFO [master/80666a11a09f:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-05T12:01:10,100 INFO [master/80666a11a09f:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-05T12:01:10,100 INFO [master/80666a11a09f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-05T12:01:10,104 INFO [master/80666a11a09f:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T12:01:10,105 INFO [master/80666a11a09f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-05T12:01:10,113 INFO [master/80666a11a09f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-12-05T12:01:10,113 INFO [master/80666a11a09f:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-05T12:01:10,162 INFO [master/80666a11a09f:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-05T12:01:10,177 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41433-0x101a6a79fd00000, quorum=127.0.0.1:51837, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-05T12:01:10,220 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-05T12:01:10,223 INFO [master/80666a11a09f:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-05T12:01:10,225 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41433-0x101a6a79fd00000, quorum=127.0.0.1:51837, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-05T12:01:10,229 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-05T12:01:10,232 INFO [master/80666a11a09f:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-05T12:01:10,237 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41433-0x101a6a79fd00000, quorum=127.0.0.1:51837, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-05T12:01:10,246 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-05T12:01:10,248 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41433-0x101a6a79fd00000, quorum=127.0.0.1:51837, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-05T12:01:10,254 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-05T12:01:10,275 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41433-0x101a6a79fd00000, quorum=127.0.0.1:51837, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-05T12:01:10,286 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-05T12:01:10,305 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41433-0x101a6a79fd00000, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T12:01:10,305 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33013-0x101a6a79fd00002, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T12:01:10,305 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34899-0x101a6a79fd00001, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T12:01:10,305 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34899-0x101a6a79fd00001, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:01:10,305 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41433-0x101a6a79fd00000, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:01:10,306 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33013-0x101a6a79fd00002, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:01:10,310 INFO [master/80666a11a09f:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=80666a11a09f,41433,1733400067757, sessionid=0x101a6a79fd00000, setting cluster-up flag (Was=false) 2024-12-05T12:01:10,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41433-0x101a6a79fd00000, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:01:10,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33013-0x101a6a79fd00002, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:01:10,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34899-0x101a6a79fd00001, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:01:10,363 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-05T12:01:10,365 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=80666a11a09f,41433,1733400067757 2024-12-05T12:01:10,380 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33013-0x101a6a79fd00002, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:01:10,380 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34899-0x101a6a79fd00001, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:01:10,380 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41433-0x101a6a79fd00000, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:01:10,405 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-05T12:01:10,407 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=80666a11a09f,41433,1733400067757 2024-12-05T12:01:10,412 INFO [master/80666a11a09f:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-05T12:01:10,429 INFO [RS:0;80666a11a09f:34899 {}] regionserver.HRegionServer(746): ClusterId : 8c4ff262-9bf1-4aef-8074-a07876e58646 2024-12-05T12:01:10,429 INFO [RS:1;80666a11a09f:33013 {}] regionserver.HRegionServer(746): ClusterId : 8c4ff262-9bf1-4aef-8074-a07876e58646 2024-12-05T12:01:10,432 DEBUG [RS:1;80666a11a09f:33013 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-05T12:01:10,432 DEBUG [RS:0;80666a11a09f:34899 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-05T12:01:10,441 DEBUG [RS:0;80666a11a09f:34899 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-05T12:01:10,441 DEBUG [RS:1;80666a11a09f:33013 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-05T12:01:10,441 DEBUG [RS:0;80666a11a09f:34899 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-05T12:01:10,441 DEBUG [RS:1;80666a11a09f:33013 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-05T12:01:10,448 DEBUG [RS:0;80666a11a09f:34899 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-05T12:01:10,448 DEBUG [RS:1;80666a11a09f:33013 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-05T12:01:10,449 DEBUG [RS:1;80666a11a09f:33013 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@213a2ff5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=80666a11a09f/172.17.0.2:0 2024-12-05T12:01:10,449 DEBUG [RS:0;80666a11a09f:34899 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50477c62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=80666a11a09f/172.17.0.2:0 2024-12-05T12:01:10,471 DEBUG [RS:0;80666a11a09f:34899 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;80666a11a09f:34899 2024-12-05T12:01:10,475 INFO [RS:0;80666a11a09f:34899 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-05T12:01:10,475 INFO [RS:0;80666a11a09f:34899 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-05T12:01:10,475 DEBUG [RS:1;80666a11a09f:33013 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;80666a11a09f:33013 2024-12-05T12:01:10,475 DEBUG [RS:0;80666a11a09f:34899 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-05T12:01:10,476 INFO [RS:1;80666a11a09f:33013 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-05T12:01:10,476 INFO [RS:1;80666a11a09f:33013 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-05T12:01:10,476 DEBUG [RS:1;80666a11a09f:33013 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-05T12:01:10,478 INFO [RS:1;80666a11a09f:33013 {}] regionserver.HRegionServer(2659): reportForDuty to master=80666a11a09f,41433,1733400067757 with port=33013, startcode=1733400068967 2024-12-05T12:01:10,479 INFO [RS:0;80666a11a09f:34899 {}] regionserver.HRegionServer(2659): reportForDuty to master=80666a11a09f,41433,1733400067757 with port=34899, startcode=1733400068760 2024-12-05T12:01:10,495 DEBUG [RS:1;80666a11a09f:33013 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-05T12:01:10,495 DEBUG [RS:0;80666a11a09f:34899 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-05T12:01:10,504 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T12:01:10,515 INFO [master/80666a11a09f:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-05T12:01:10,524 INFO [master/80666a11a09f:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-05T12:01:10,530 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 80666a11a09f,41433,1733400067757 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-05T12:01:10,539 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/80666a11a09f:0, corePoolSize=5, maxPoolSize=5 2024-12-05T12:01:10,539 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/80666a11a09f:0, corePoolSize=5, maxPoolSize=5 2024-12-05T12:01:10,540 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/80666a11a09f:0, corePoolSize=5, maxPoolSize=5 2024-12-05T12:01:10,540 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/80666a11a09f:0, corePoolSize=5, maxPoolSize=5 2024-12-05T12:01:10,540 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/80666a11a09f:0, corePoolSize=10, maxPoolSize=10 2024-12-05T12:01:10,541 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:10,541 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/80666a11a09f:0, corePoolSize=2, maxPoolSize=2 2024-12-05T12:01:10,541 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:10,580 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T12:01:10,583 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-05T12:01:10,584 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55903, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-05T12:01:10,584 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47829, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-05T12:01:10,590 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:01:10,591 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-05T12:01:10,593 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41433 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T12:01:10,599 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41433 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-05T12:01:10,603 INFO [master/80666a11a09f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733400100603 2024-12-05T12:01:10,606 INFO [master/80666a11a09f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-05T12:01:10,607 INFO [master/80666a11a09f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-05T12:01:10,611 INFO [master/80666a11a09f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-05T12:01:10,612 INFO [master/80666a11a09f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-05T12:01:10,612 INFO [master/80666a11a09f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-05T12:01:10,612 INFO [master/80666a11a09f:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-05T12:01:10,619 INFO [master/80666a11a09f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:10,627 INFO [master/80666a11a09f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-05T12:01:10,628 DEBUG [RS:0;80666a11a09f:34899 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-05T12:01:10,628 WARN [RS:0;80666a11a09f:34899 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-05T12:01:10,628 DEBUG [RS:1;80666a11a09f:33013 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-05T12:01:10,628 WARN [RS:1;80666a11a09f:33013 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-05T12:01:10,629 INFO [master/80666a11a09f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-05T12:01:10,629 INFO [master/80666a11a09f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-05T12:01:10,635 INFO [master/80666a11a09f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-05T12:01:10,635 INFO [master/80666a11a09f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-05T12:01:10,639 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/80666a11a09f:0:becomeActiveMaster-HFileCleaner.large.0-1733400070636,5,FailOnTimeoutGroup] 2024-12-05T12:01:10,647 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/80666a11a09f:0:becomeActiveMaster-HFileCleaner.small.0-1733400070639,5,FailOnTimeoutGroup] 2024-12-05T12:01:10,647 INFO [master/80666a11a09f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:10,648 INFO [master/80666a11a09f:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-05T12:01:10,649 INFO [master/80666a11a09f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:10,650 INFO [master/80666a11a09f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:10,651 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-05T12:01:10,652 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6 2024-12-05T12:01:10,673 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T12:01:10,675 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-05T12:01:10,680 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T12:01:10,681 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:01:10,681 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T12:01:10,681 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-05T12:01:10,688 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-05T12:01:10,688 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:01:10,688 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T12:01:10,689 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-05T12:01:10,693 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T12:01:10,693 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:01:10,694 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T12:01:10,694 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-05T12:01:10,700 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T12:01:10,701 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:01:10,701 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T12:01:10,701 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-05T12:01:10,702 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740 2024-12-05T12:01:10,703 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740 2024-12-05T12:01:10,703 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-05T12:01:10,703 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-05T12:01:10,704 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T12:01:10,705 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-05T12:01:10,713 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T12:01:10,714 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62116910, jitterRate=-0.07438591122627258}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T12:01:10,718 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733400070673Initializing all the Stores at 1733400070674 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733400070674Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733400070674Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733400070675 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733400070675Cleaning up temporary data from old regions at 1733400070703 (+28 ms)Region opened successfully at 1733400070718 (+15 ms) 2024-12-05T12:01:10,718 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-05T12:01:10,718 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-05T12:01:10,718 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-05T12:01:10,718 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T12:01:10,719 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T12:01:10,720 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-05T12:01:10,721 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733400070718Disabling compacts and flushes for region at 1733400070718Disabling writes for close at 1733400070719 (+1 ms)Writing region close event to WAL at 1733400070720 (+1 ms)Closed at 1733400070720 2024-12-05T12:01:10,724 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T12:01:10,724 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-05T12:01:10,730 INFO [RS:1;80666a11a09f:33013 {}] regionserver.HRegionServer(2659): reportForDuty to master=80666a11a09f,41433,1733400067757 with port=33013, startcode=1733400068967 2024-12-05T12:01:10,730 INFO [RS:0;80666a11a09f:34899 {}] regionserver.HRegionServer(2659): reportForDuty to master=80666a11a09f,41433,1733400067757 with port=34899, startcode=1733400068760 2024-12-05T12:01:10,732 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-05T12:01:10,732 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41433 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 80666a11a09f,34899,1733400068760 2024-12-05T12:01:10,735 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41433 {}] master.ServerManager(517): Registering regionserver=80666a11a09f,34899,1733400068760 2024-12-05T12:01:10,742 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T12:01:10,744 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-05T12:01:10,745 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41433 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 80666a11a09f,33013,1733400068967 2024-12-05T12:01:10,745 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41433 {}] master.ServerManager(517): Registering regionserver=80666a11a09f,33013,1733400068967 2024-12-05T12:01:10,747 DEBUG [RS:0;80666a11a09f:34899 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6 2024-12-05T12:01:10,747 DEBUG [RS:0;80666a11a09f:34899 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-05T12:01:10,747 DEBUG [RS:0;80666a11a09f:34899 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-05T12:01:10,753 DEBUG [RS:1;80666a11a09f:33013 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6 2024-12-05T12:01:10,753 DEBUG [RS:1;80666a11a09f:33013 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-05T12:01:10,753 DEBUG [RS:1;80666a11a09f:33013 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-05T12:01:10,895 WARN [80666a11a09f:41433 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T12:01:11,071 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41433-0x101a6a79fd00000, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T12:01:11,104 DEBUG [RS:0;80666a11a09f:34899 {}] zookeeper.ZKUtil(111): regionserver:34899-0x101a6a79fd00001, quorum=127.0.0.1:51837, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/80666a11a09f,34899,1733400068760 2024-12-05T12:01:11,104 WARN [RS:0;80666a11a09f:34899 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T12:01:11,104 DEBUG [RS:1;80666a11a09f:33013 {}] zookeeper.ZKUtil(111): regionserver:33013-0x101a6a79fd00002, quorum=127.0.0.1:51837, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/80666a11a09f,33013,1733400068967 2024-12-05T12:01:11,105 INFO [RS:0;80666a11a09f:34899 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T12:01:11,105 WARN [RS:1;80666a11a09f:33013 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T12:01:11,105 INFO [RS:1;80666a11a09f:33013 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T12:01:11,105 DEBUG [RS:0;80666a11a09f:34899 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/WALs/80666a11a09f,34899,1733400068760 2024-12-05T12:01:11,105 DEBUG [RS:1;80666a11a09f:33013 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/WALs/80666a11a09f,33013,1733400068967 2024-12-05T12:01:11,106 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [80666a11a09f,33013,1733400068967] 2024-12-05T12:01:11,106 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [80666a11a09f,34899,1733400068760] 2024-12-05T12:01:11,129 INFO [RS:0;80666a11a09f:34899 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-05T12:01:11,129 INFO [RS:1;80666a11a09f:33013 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-05T12:01:11,146 INFO [RS:0;80666a11a09f:34899 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-05T12:01:11,147 INFO [RS:1;80666a11a09f:33013 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-05T12:01:11,153 INFO [RS:1;80666a11a09f:33013 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-05T12:01:11,153 INFO [RS:0;80666a11a09f:34899 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-05T12:01:11,153 INFO [RS:1;80666a11a09f:33013 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:11,153 INFO [RS:0;80666a11a09f:34899 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:11,154 INFO [RS:1;80666a11a09f:33013 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-05T12:01:11,154 INFO [RS:0;80666a11a09f:34899 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-05T12:01:11,159 INFO [RS:1;80666a11a09f:33013 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-05T12:01:11,159 INFO [RS:0;80666a11a09f:34899 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-05T12:01:11,161 INFO [RS:0;80666a11a09f:34899 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:11,161 INFO [RS:1;80666a11a09f:33013 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:11,161 DEBUG [RS:0;80666a11a09f:34899 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:11,161 DEBUG [RS:1;80666a11a09f:33013 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:11,162 DEBUG [RS:1;80666a11a09f:33013 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:11,162 DEBUG [RS:0;80666a11a09f:34899 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:11,162 DEBUG [RS:1;80666a11a09f:33013 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:11,162 DEBUG [RS:0;80666a11a09f:34899 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:11,162 DEBUG [RS:1;80666a11a09f:33013 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:11,162 DEBUG [RS:0;80666a11a09f:34899 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:11,162 DEBUG [RS:1;80666a11a09f:33013 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:11,162 DEBUG [RS:0;80666a11a09f:34899 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:11,162 DEBUG [RS:0;80666a11a09f:34899 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/80666a11a09f:0, corePoolSize=2, maxPoolSize=2 2024-12-05T12:01:11,162 DEBUG [RS:1;80666a11a09f:33013 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/80666a11a09f:0, corePoolSize=2, maxPoolSize=2 2024-12-05T12:01:11,162 DEBUG [RS:1;80666a11a09f:33013 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:11,163 DEBUG [RS:1;80666a11a09f:33013 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:11,163 DEBUG [RS:0;80666a11a09f:34899 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:11,163 DEBUG [RS:1;80666a11a09f:33013 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:11,163 DEBUG [RS:0;80666a11a09f:34899 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:11,163 DEBUG [RS:1;80666a11a09f:33013 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:11,163 DEBUG [RS:0;80666a11a09f:34899 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:11,163 DEBUG [RS:1;80666a11a09f:33013 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:11,163 DEBUG [RS:0;80666a11a09f:34899 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:11,163 DEBUG [RS:1;80666a11a09f:33013 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:11,163 DEBUG [RS:0;80666a11a09f:34899 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:11,164 DEBUG [RS:1;80666a11a09f:33013 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/80666a11a09f:0, corePoolSize=3, maxPoolSize=3 2024-12-05T12:01:11,164 DEBUG [RS:0;80666a11a09f:34899 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:11,164 DEBUG [RS:0;80666a11a09f:34899 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/80666a11a09f:0, corePoolSize=3, maxPoolSize=3 2024-12-05T12:01:11,164 DEBUG [RS:0;80666a11a09f:34899 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0, corePoolSize=3, maxPoolSize=3 2024-12-05T12:01:11,164 DEBUG [RS:1;80666a11a09f:33013 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0, corePoolSize=3, maxPoolSize=3 2024-12-05T12:01:11,171 INFO [RS:0;80666a11a09f:34899 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:11,171 INFO [RS:0;80666a11a09f:34899 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:11,171 INFO [RS:0;80666a11a09f:34899 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:11,171 INFO [RS:0;80666a11a09f:34899 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:11,171 INFO [RS:0;80666a11a09f:34899 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:11,171 INFO [RS:0;80666a11a09f:34899 {}] hbase.ChoreService(168): Chore ScheduledChore name=80666a11a09f,34899,1733400068760-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T12:01:11,174 INFO [RS:1;80666a11a09f:33013 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:11,175 INFO [RS:1;80666a11a09f:33013 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:11,175 INFO [RS:1;80666a11a09f:33013 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:11,175 INFO [RS:1;80666a11a09f:33013 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:11,175 INFO [RS:1;80666a11a09f:33013 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:11,175 INFO [RS:1;80666a11a09f:33013 {}] hbase.ChoreService(168): Chore ScheduledChore name=80666a11a09f,33013,1733400068967-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T12:01:11,194 INFO [RS:0;80666a11a09f:34899 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-05T12:01:11,196 INFO [RS:0;80666a11a09f:34899 {}] hbase.ChoreService(168): Chore ScheduledChore name=80666a11a09f,34899,1733400068760-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:11,197 INFO [RS:0;80666a11a09f:34899 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:11,197 INFO [RS:0;80666a11a09f:34899 {}] regionserver.Replication(171): 80666a11a09f,34899,1733400068760 started 2024-12-05T12:01:11,198 INFO [RS:1;80666a11a09f:33013 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-05T12:01:11,199 INFO [RS:1;80666a11a09f:33013 {}] hbase.ChoreService(168): Chore ScheduledChore name=80666a11a09f,33013,1733400068967-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:11,199 INFO [RS:1;80666a11a09f:33013 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:11,199 INFO [RS:1;80666a11a09f:33013 {}] regionserver.Replication(171): 80666a11a09f,33013,1733400068967 started 2024-12-05T12:01:11,217 INFO [RS:0;80666a11a09f:34899 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:11,218 INFO [RS:0;80666a11a09f:34899 {}] regionserver.HRegionServer(1482): Serving as 80666a11a09f,34899,1733400068760, RpcServer on 80666a11a09f/172.17.0.2:34899, sessionid=0x101a6a79fd00001 2024-12-05T12:01:11,219 DEBUG [RS:0;80666a11a09f:34899 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-05T12:01:11,219 DEBUG [RS:0;80666a11a09f:34899 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 80666a11a09f,34899,1733400068760 2024-12-05T12:01:11,219 DEBUG [RS:0;80666a11a09f:34899 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '80666a11a09f,34899,1733400068760' 2024-12-05T12:01:11,220 DEBUG [RS:0;80666a11a09f:34899 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-05T12:01:11,221 DEBUG [RS:0;80666a11a09f:34899 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-05T12:01:11,222 DEBUG [RS:0;80666a11a09f:34899 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-05T12:01:11,222 DEBUG [RS:0;80666a11a09f:34899 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-05T12:01:11,222 DEBUG [RS:0;80666a11a09f:34899 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 80666a11a09f,34899,1733400068760 2024-12-05T12:01:11,222 DEBUG [RS:0;80666a11a09f:34899 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '80666a11a09f,34899,1733400068760' 2024-12-05T12:01:11,222 INFO [RS:1;80666a11a09f:33013 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:11,222 DEBUG [RS:0;80666a11a09f:34899 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-05T12:01:11,222 INFO [RS:1;80666a11a09f:33013 {}] regionserver.HRegionServer(1482): Serving as 80666a11a09f,33013,1733400068967, RpcServer on 80666a11a09f/172.17.0.2:33013, sessionid=0x101a6a79fd00002 2024-12-05T12:01:11,223 DEBUG [RS:1;80666a11a09f:33013 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-05T12:01:11,223 DEBUG [RS:1;80666a11a09f:33013 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 80666a11a09f,33013,1733400068967 2024-12-05T12:01:11,223 DEBUG [RS:1;80666a11a09f:33013 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '80666a11a09f,33013,1733400068967' 2024-12-05T12:01:11,223 DEBUG [RS:1;80666a11a09f:33013 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-05T12:01:11,223 DEBUG [RS:0;80666a11a09f:34899 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-05T12:01:11,224 DEBUG [RS:1;80666a11a09f:33013 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-05T12:01:11,224 DEBUG [RS:0;80666a11a09f:34899 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-05T12:01:11,224 INFO [RS:0;80666a11a09f:34899 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-05T12:01:11,224 INFO [RS:0;80666a11a09f:34899 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-05T12:01:11,224 DEBUG [RS:1;80666a11a09f:33013 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-05T12:01:11,224 DEBUG [RS:1;80666a11a09f:33013 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-05T12:01:11,224 DEBUG [RS:1;80666a11a09f:33013 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 80666a11a09f,33013,1733400068967 2024-12-05T12:01:11,225 DEBUG [RS:1;80666a11a09f:33013 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '80666a11a09f,33013,1733400068967' 2024-12-05T12:01:11,225 DEBUG [RS:1;80666a11a09f:33013 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-05T12:01:11,225 DEBUG [RS:1;80666a11a09f:33013 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-05T12:01:11,226 DEBUG [RS:1;80666a11a09f:33013 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-05T12:01:11,226 INFO [RS:1;80666a11a09f:33013 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-05T12:01:11,226 INFO [RS:1;80666a11a09f:33013 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-05T12:01:11,330 INFO [RS:1;80666a11a09f:33013 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T12:01:11,330 INFO [RS:0;80666a11a09f:34899 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T12:01:11,331 INFO [RS:0;80666a11a09f:34899 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=80666a11a09f%2C34899%2C1733400068760, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/WALs/80666a11a09f,34899,1733400068760, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/oldWALs, maxLogs=55 2024-12-05T12:01:11,332 INFO [RS:1;80666a11a09f:33013 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=80666a11a09f%2C33013%2C1733400068967, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/WALs/80666a11a09f,33013,1733400068967, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/oldWALs, maxLogs=55 2024-12-05T12:01:11,354 INFO [RS:1;80666a11a09f:33013 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/WALs/80666a11a09f,33013,1733400068967/80666a11a09f%2C33013%2C1733400068967.1733400071335 2024-12-05T12:01:11,355 DEBUG [RS:1;80666a11a09f:33013 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-05T12:01:11,358 INFO [RS:0;80666a11a09f:34899 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/WALs/80666a11a09f,34899,1733400068760/80666a11a09f%2C34899%2C1733400068760.1733400071335 2024-12-05T12:01:11,358 DEBUG [RS:0;80666a11a09f:34899 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-05T12:01:11,398 DEBUG [80666a11a09f:41433 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-05T12:01:11,408 DEBUG [80666a11a09f:41433 {}] balancer.BalancerClusterState(204): Hosts are {80666a11a09f=0} racks are {/default-rack=0} 2024-12-05T12:01:11,411 DEBUG [80666a11a09f:41433 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-05T12:01:11,411 DEBUG [80666a11a09f:41433 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-05T12:01:11,411 DEBUG [80666a11a09f:41433 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-05T12:01:11,411 DEBUG [80666a11a09f:41433 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-05T12:01:11,412 INFO [80666a11a09f:41433 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-05T12:01:11,412 INFO [80666a11a09f:41433 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-05T12:01:11,412 DEBUG [80666a11a09f:41433 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-05T12:01:11,419 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=80666a11a09f,33013,1733400068967 2024-12-05T12:01:11,424 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 80666a11a09f,33013,1733400068967, state=OPENING 2024-12-05T12:01:11,438 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-05T12:01:11,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34899-0x101a6a79fd00001, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:01:11,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41433-0x101a6a79fd00000, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:01:11,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33013-0x101a6a79fd00002, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:01:11,447 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T12:01:11,447 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T12:01:11,447 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T12:01:11,449 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T12:01:11,451 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=80666a11a09f,33013,1733400068967}] 2024-12-05T12:01:11,623 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-05T12:01:11,626 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39659, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-05T12:01:11,635 INFO [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-05T12:01:11,635 INFO [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T12:01:11,637 INFO [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-05T12:01:11,638 INFO [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=80666a11a09f%2C33013%2C1733400068967.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/WALs/80666a11a09f,33013,1733400068967, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/oldWALs, maxLogs=55 2024-12-05T12:01:11,663 INFO [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/WALs/80666a11a09f,33013,1733400068967/80666a11a09f%2C33013%2C1733400068967.meta.1733400071640.meta 2024-12-05T12:01:11,666 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-05T12:01:11,666 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-05T12:01:11,668 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-05T12:01:11,670 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-05T12:01:11,675 INFO [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-05T12:01:11,679 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-05T12:01:11,680 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T12:01:11,680 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-05T12:01:11,680 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-05T12:01:11,683 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-05T12:01:11,684 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T12:01:11,684 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:01:11,684 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T12:01:11,684 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-05T12:01:11,685 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-05T12:01:11,685 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:01:11,685 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T12:01:11,686 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-05T12:01:11,687 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T12:01:11,687 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:01:11,687 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T12:01:11,687 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-05T12:01:11,688 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T12:01:11,688 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:01:11,688 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T12:01:11,689 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-05T12:01:11,690 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740 2024-12-05T12:01:11,691 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740 2024-12-05T12:01:11,692 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-05T12:01:11,692 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-05T12:01:11,692 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T12:01:11,694 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-05T12:01:11,695 INFO [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63026071, jitterRate=-0.060838356614112854}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T12:01:11,695 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-05T12:01:11,697 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733400071681Writing region info on filesystem at 1733400071681Initializing all the Stores at 1733400071682 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733400071682Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733400071682Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733400071682Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733400071682Cleaning up temporary data from old regions at 1733400071692 (+10 ms)Running coprocessor post-open hooks at 1733400071696 (+4 ms)Region opened successfully at 1733400071697 (+1 ms) 2024-12-05T12:01:11,703 INFO [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733400071615 2024-12-05T12:01:11,713 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-05T12:01:11,714 INFO [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-05T12:01:11,715 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=80666a11a09f,33013,1733400068967 2024-12-05T12:01:11,717 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 80666a11a09f,33013,1733400068967, state=OPEN 2024-12-05T12:01:11,761 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33013-0x101a6a79fd00002, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T12:01:11,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34899-0x101a6a79fd00001, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T12:01:11,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41433-0x101a6a79fd00000, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T12:01:11,762 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T12:01:11,762 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T12:01:11,762 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T12:01:11,763 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=80666a11a09f,33013,1733400068967 2024-12-05T12:01:11,767 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-05T12:01:11,767 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=80666a11a09f,33013,1733400068967 in 312 msec 2024-12-05T12:01:11,773 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-05T12:01:11,773 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.0370 sec 2024-12-05T12:01:11,774 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T12:01:11,774 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-05T12:01:11,802 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:01:11,804 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,33013,1733400068967, seqNum=-1] 2024-12-05T12:01:11,823 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:01:11,826 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53621, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:01:11,847 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.4010 sec 2024-12-05T12:01:11,847 INFO [master/80666a11a09f:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733400071847, completionTime=-1 2024-12-05T12:01:11,851 INFO [master/80666a11a09f:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-05T12:01:11,851 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T12:01:11,881 INFO [master/80666a11a09f:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-05T12:01:11,881 INFO [master/80666a11a09f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733400131881 2024-12-05T12:01:11,882 INFO [master/80666a11a09f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733400191882 2024-12-05T12:01:11,882 INFO [master/80666a11a09f:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 30 msec 2024-12-05T12:01:11,885 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-05T12:01:11,892 INFO [master/80666a11a09f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80666a11a09f,41433,1733400067757-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:11,893 INFO [master/80666a11a09f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80666a11a09f,41433,1733400067757-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:11,893 INFO [master/80666a11a09f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80666a11a09f,41433,1733400067757-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:11,894 INFO [master/80666a11a09f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-80666a11a09f:41433, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:11,895 INFO [master/80666a11a09f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:11,895 INFO [master/80666a11a09f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:11,902 DEBUG [master/80666a11a09f:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-05T12:01:11,927 INFO [master/80666a11a09f:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.686sec 2024-12-05T12:01:11,929 INFO [master/80666a11a09f:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-05T12:01:11,930 INFO [master/80666a11a09f:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-05T12:01:11,931 INFO [master/80666a11a09f:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-05T12:01:11,932 INFO [master/80666a11a09f:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-05T12:01:11,932 INFO [master/80666a11a09f:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-05T12:01:11,933 INFO [master/80666a11a09f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80666a11a09f,41433,1733400067757-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T12:01:11,933 INFO [master/80666a11a09f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80666a11a09f,41433,1733400067757-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-05T12:01:11,934 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-05T12:01:11,935 INFO [master/80666a11a09f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-05T12:01:11,936 INFO [master/80666a11a09f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80666a11a09f,41433,1733400067757-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:11,940 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6098c9c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:01:11,943 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-05T12:01:11,943 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-05T12:01:11,947 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,41433,-1 for getting cluster id 2024-12-05T12:01:11,950 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:01:11,957 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8c4ff262-9bf1-4aef-8074-a07876e58646' 2024-12-05T12:01:11,959 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:01:11,960 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8c4ff262-9bf1-4aef-8074-a07876e58646" 2024-12-05T12:01:11,962 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@acb0ed7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:01:11,962 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,41433,-1] 2024-12-05T12:01:11,964 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:01:11,966 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:01:11,967 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48742, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:01:11,970 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67ae5fc1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:01:11,971 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:01:11,979 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,33013,1733400068967, seqNum=-1] 2024-12-05T12:01:11,979 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:01:11,982 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46118, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:01:12,023 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=80666a11a09f,41433,1733400067757 2024-12-05T12:01:12,031 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T12:01:12,040 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(321): The fetched master address is 80666a11a09f,41433,1733400067757 2024-12-05T12:01:12,043 DEBUG [Time-limited test {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7a17f38d 2024-12-05T12:01:12,046 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T12:01:12,055 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48746, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T12:01:12,063 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41433 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1733400072023', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T12:01:12,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41433 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1733400072023 2024-12-05T12:01:12,075 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733400072023 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T12:01:12,077 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41433 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1733400072023" procId is: 4 2024-12-05T12:01:12,079 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:01:12,082 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733400072023 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T12:01:12,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T12:01:12,105 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733400072023-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 940307e0d31686aca88e50f913c461c3, NAME => 'testBlockEvictionOnRegionMove1733400072023,,1733400072057.940307e0d31686aca88e50f913c461c3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1733400072023', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6 2024-12-05T12:01:12,112 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733400072023-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733400072023,,1733400072057.940307e0d31686aca88e50f913c461c3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T12:01:12,112 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733400072023-pool-0 {}] regionserver.HRegion(1722): Closing 940307e0d31686aca88e50f913c461c3, disabling compactions & flushes 2024-12-05T12:01:12,112 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733400072023-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733400072023,,1733400072057.940307e0d31686aca88e50f913c461c3. 2024-12-05T12:01:12,112 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733400072023-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733400072023,,1733400072057.940307e0d31686aca88e50f913c461c3. 2024-12-05T12:01:12,112 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733400072023-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733400072023,,1733400072057.940307e0d31686aca88e50f913c461c3. after waiting 0 ms 2024-12-05T12:01:12,112 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733400072023-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733400072023,,1733400072057.940307e0d31686aca88e50f913c461c3. 2024-12-05T12:01:12,112 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733400072023-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733400072023,,1733400072057.940307e0d31686aca88e50f913c461c3. 2024-12-05T12:01:12,112 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733400072023-pool-0 {}] regionserver.HRegion(1676): Region close journal for 940307e0d31686aca88e50f913c461c3: Waiting for close lock at 1733400072112Disabling compacts and flushes for region at 1733400072112Disabling writes for close at 1733400072112Writing region close event to WAL at 1733400072112Closed at 1733400072112 2024-12-05T12:01:12,114 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733400072023 execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T12:01:12,118 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1733400072023,,1733400072057.940307e0d31686aca88e50f913c461c3.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733400072114"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733400072114"}]},"ts":"1733400072114"} 2024-12-05T12:01:12,123 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T12:01:12,124 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733400072023 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T12:01:12,127 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1733400072023","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733400072125"}]},"ts":"1733400072125"} 2024-12-05T12:01:12,131 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1733400072023, state=ENABLING in hbase:meta 2024-12-05T12:01:12,132 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {80666a11a09f=0} racks are {/default-rack=0} 2024-12-05T12:01:12,133 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-05T12:01:12,133 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-05T12:01:12,133 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-05T12:01:12,133 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-05T12:01:12,133 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-05T12:01:12,133 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-05T12:01:12,133 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-05T12:01:12,135 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733400072023, region=940307e0d31686aca88e50f913c461c3, ASSIGN}] 2024-12-05T12:01:12,137 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733400072023, region=940307e0d31686aca88e50f913c461c3, ASSIGN 2024-12-05T12:01:12,138 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733400072023, region=940307e0d31686aca88e50f913c461c3, ASSIGN; state=OFFLINE, location=80666a11a09f,34899,1733400068760; forceNewPlan=false, retain=false 2024-12-05T12:01:12,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T12:01:12,290 INFO [80666a11a09f:41433 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-05T12:01:12,291 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=940307e0d31686aca88e50f913c461c3, regionState=OPENING, regionLocation=80666a11a09f,34899,1733400068760 2024-12-05T12:01:12,295 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733400072023, region=940307e0d31686aca88e50f913c461c3, ASSIGN because future has completed 2024-12-05T12:01:12,296 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 940307e0d31686aca88e50f913c461c3, server=80666a11a09f,34899,1733400068760}] 2024-12-05T12:01:12,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T12:01:12,450 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-05T12:01:12,452 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57269, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-05T12:01:12,467 INFO [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1733400072023,,1733400072057.940307e0d31686aca88e50f913c461c3. 2024-12-05T12:01:12,467 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 940307e0d31686aca88e50f913c461c3, NAME => 'testBlockEvictionOnRegionMove1733400072023,,1733400072057.940307e0d31686aca88e50f913c461c3.', STARTKEY => '', ENDKEY => ''} 2024-12-05T12:01:12,468 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1733400072023 940307e0d31686aca88e50f913c461c3 2024-12-05T12:01:12,468 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733400072023,,1733400072057.940307e0d31686aca88e50f913c461c3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T12:01:12,468 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 940307e0d31686aca88e50f913c461c3 2024-12-05T12:01:12,469 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 940307e0d31686aca88e50f913c461c3 2024-12-05T12:01:12,471 INFO [StoreOpener-940307e0d31686aca88e50f913c461c3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 940307e0d31686aca88e50f913c461c3 2024-12-05T12:01:12,487 INFO [StoreOpener-940307e0d31686aca88e50f913c461c3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 940307e0d31686aca88e50f913c461c3 columnFamilyName family 2024-12-05T12:01:12,487 DEBUG [StoreOpener-940307e0d31686aca88e50f913c461c3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:01:12,488 INFO [StoreOpener-940307e0d31686aca88e50f913c461c3-1 {}] regionserver.HStore(327): Store=940307e0d31686aca88e50f913c461c3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T12:01:12,489 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 940307e0d31686aca88e50f913c461c3 2024-12-05T12:01:12,490 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/default/testBlockEvictionOnRegionMove1733400072023/940307e0d31686aca88e50f913c461c3 2024-12-05T12:01:12,491 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/default/testBlockEvictionOnRegionMove1733400072023/940307e0d31686aca88e50f913c461c3 2024-12-05T12:01:12,491 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 940307e0d31686aca88e50f913c461c3 2024-12-05T12:01:12,491 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 940307e0d31686aca88e50f913c461c3 2024-12-05T12:01:12,493 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 940307e0d31686aca88e50f913c461c3 2024-12-05T12:01:12,503 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/default/testBlockEvictionOnRegionMove1733400072023/940307e0d31686aca88e50f913c461c3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T12:01:12,503 INFO [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 940307e0d31686aca88e50f913c461c3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62959164, jitterRate=-0.06183534860610962}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T12:01:12,503 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 940307e0d31686aca88e50f913c461c3 2024-12-05T12:01:12,504 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 940307e0d31686aca88e50f913c461c3: Running coprocessor pre-open hook at 1733400072469Writing region info on filesystem at 1733400072469Initializing all the Stores at 1733400072469Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733400072469Cleaning up temporary data from old regions at 1733400072492 (+23 ms)Running coprocessor post-open hooks at 1733400072503 (+11 ms)Region opened successfully at 1733400072504 (+1 ms) 2024-12-05T12:01:12,506 INFO [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1733400072023,,1733400072057.940307e0d31686aca88e50f913c461c3., pid=6, masterSystemTime=1733400072449 2024-12-05T12:01:12,509 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1733400072023,,1733400072057.940307e0d31686aca88e50f913c461c3. 2024-12-05T12:01:12,510 INFO [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1733400072023,,1733400072057.940307e0d31686aca88e50f913c461c3. 2024-12-05T12:01:12,510 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=940307e0d31686aca88e50f913c461c3, regionState=OPEN, openSeqNum=2, regionLocation=80666a11a09f,34899,1733400068760 2024-12-05T12:01:12,514 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 940307e0d31686aca88e50f913c461c3, server=80666a11a09f,34899,1733400068760 because future has completed 2024-12-05T12:01:12,517 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-05T12:01:12,518 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 940307e0d31686aca88e50f913c461c3, server=80666a11a09f,34899,1733400068760 in 219 msec 2024-12-05T12:01:12,519 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-05T12:01:12,520 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733400072023, region=940307e0d31686aca88e50f913c461c3, ASSIGN in 382 msec 2024-12-05T12:01:12,520 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733400072023 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T12:01:12,521 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1733400072023","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733400072520"}]},"ts":"1733400072520"} 2024-12-05T12:01:12,524 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1733400072023, state=ENABLED in hbase:meta 2024-12-05T12:01:12,526 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733400072023 execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T12:01:12,529 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1733400072023 in 458 msec 2024-12-05T12:01:12,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T12:01:12,716 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1733400072023 get assigned. Timeout = 60000ms 2024-12-05T12:01:12,716 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1733400072023 completed 2024-12-05T12:01:12,719 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T12:01:12,725 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1733400072023 assigned to meta. Checking AM states. 2024-12-05T12:01:12,726 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T12:01:12,726 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1733400072023 assigned. 2024-12-05T12:01:12,736 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1733400072023', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1733400072023,,1733400072057.940307e0d31686aca88e50f913c461c3., hostname=80666a11a09f,34899,1733400068760, seqNum=2] 2024-12-05T12:01:12,737 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:01:12,740 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58212, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:01:12,755 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 940307e0d31686aca88e50f913c461c3 1/1 column families, dataSize=78 B heapSize=480 B 2024-12-05T12:01:12,783 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 95ced1f5aa3e4ecdb06d33025aa5f3fb 2024-12-05T12:01:12,830 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/default/testBlockEvictionOnRegionMove1733400072023/940307e0d31686aca88e50f913c461c3/.tmp/family/95ced1f5aa3e4ecdb06d33025aa5f3fb is 43, key is row1/family:qf1/1/Put/seqid=0 2024-12-05T12:01:12,834 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/default/testBlockEvictionOnRegionMove1733400072023/940307e0d31686aca88e50f913c461c3/.tmp/family/95ced1f5aa3e4ecdb06d33025aa5f3fb 2024-12-05T12:01:12,868 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/default/testBlockEvictionOnRegionMove1733400072023/940307e0d31686aca88e50f913c461c3/.tmp/family/95ced1f5aa3e4ecdb06d33025aa5f3fb as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/default/testBlockEvictionOnRegionMove1733400072023/940307e0d31686aca88e50f913c461c3/family/95ced1f5aa3e4ecdb06d33025aa5f3fb 2024-12-05T12:01:12,869 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/default/testBlockEvictionOnRegionMove1733400072023/940307e0d31686aca88e50f913c461c3/family/95ced1f5aa3e4ecdb06d33025aa5f3fb, delay=947 ms 2024-12-05T12:01:12,871 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/default/testBlockEvictionOnRegionMove1733400072023/940307e0d31686aca88e50f913c461c3/family/95ced1f5aa3e4ecdb06d33025aa5f3fb, entries=2, sequenceid=6, filesize=4.9 K 2024-12-05T12:01:12,878 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 940307e0d31686aca88e50f913c461c3 in 122ms, sequenceid=6, compaction requested=false 2024-12-05T12:01:12,879 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1733400072023' 2024-12-05T12:01:12,881 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 940307e0d31686aca88e50f913c461c3: 2024-12-05T12:01:13,819 DEBUG [hfile-prefetch-1733400072870 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 95ced1f5aa3e4ecdb06d33025aa5f3fb_0. 2024-12-05T12:01:13,824 DEBUG [hfile-prefetch-1733400072870 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 95ced1f5aa3e4ecdb06d33025aa5f3fb_117. 2024-12-05T12:01:13,825 DEBUG [hfile-prefetch-1733400072870 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/default/testBlockEvictionOnRegionMove1733400072023/940307e0d31686aca88e50f913c461c3/family/95ced1f5aa3e4ecdb06d33025aa5f3fb, with total blocks 2, and data blocks 1 2024-12-05T12:01:13,825 DEBUG [hfile-prefetch-1733400072870 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-05T12:01:13,826 DEBUG [hfile-prefetch-1733400072870 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/default/testBlockEvictionOnRegionMove1733400072023/940307e0d31686aca88e50f913c461c3/family/95ced1f5aa3e4ecdb06d33025aa5f3fb, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-12-05T12:01:13,826 DEBUG [hfile-prefetch-1733400072870 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/default/testBlockEvictionOnRegionMove1733400072023/940307e0d31686aca88e50f913c461c3/family/95ced1f5aa3e4ecdb06d33025aa5f3fb. Will sleep 100ms and try the verification again. 2024-12-05T12:01:13,891 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41433 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=940307e0d31686aca88e50f913c461c3, source=80666a11a09f,34899,1733400068760, destination=80666a11a09f,33013,1733400068967, warming up region on 80666a11a09f,33013,1733400068967 2024-12-05T12:01:13,893 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41433 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=940307e0d31686aca88e50f913c461c3, source=80666a11a09f,34899,1733400068760, destination=80666a11a09f,33013,1733400068967, running balancer 2024-12-05T12:01:13,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41433 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733400072023, region=940307e0d31686aca88e50f913c461c3, REOPEN/MOVE 2024-12-05T12:01:13,895 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733400072023, region=940307e0d31686aca88e50f913c461c3, REOPEN/MOVE 2024-12-05T12:01:13,895 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33013 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1733400072023,,1733400072057.940307e0d31686aca88e50f913c461c3. 2024-12-05T12:01:13,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33013 {}] regionserver.HRegion(7855): Warmup {ENCODED => 940307e0d31686aca88e50f913c461c3, NAME => 'testBlockEvictionOnRegionMove1733400072023,,1733400072057.940307e0d31686aca88e50f913c461c3.', STARTKEY => '', ENDKEY => ''} 2024-12-05T12:01:13,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33013 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733400072023,,1733400072057.940307e0d31686aca88e50f913c461c3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T12:01:13,897 INFO [StoreOpener-940307e0d31686aca88e50f913c461c3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 940307e0d31686aca88e50f913c461c3 2024-12-05T12:01:13,898 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=940307e0d31686aca88e50f913c461c3, regionState=CLOSING, regionLocation=80666a11a09f,34899,1733400068760 2024-12-05T12:01:13,898 INFO [StoreOpener-940307e0d31686aca88e50f913c461c3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 940307e0d31686aca88e50f913c461c3 columnFamilyName family 2024-12-05T12:01:13,898 DEBUG [StoreOpener-940307e0d31686aca88e50f913c461c3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:01:13,901 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733400072023, region=940307e0d31686aca88e50f913c461c3, REOPEN/MOVE because future has completed 2024-12-05T12:01:13,903 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-12-05T12:01:13,903 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 940307e0d31686aca88e50f913c461c3, server=80666a11a09f,34899,1733400068760}] 2024-12-05T12:01:13,903 DEBUG [StoreOpener-940307e0d31686aca88e50f913c461c3-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/default/testBlockEvictionOnRegionMove1733400072023/940307e0d31686aca88e50f913c461c3/family/95ced1f5aa3e4ecdb06d33025aa5f3fb 2024-12-05T12:01:13,903 INFO [StoreOpener-940307e0d31686aca88e50f913c461c3-1 {}] regionserver.HStore(327): Store=940307e0d31686aca88e50f913c461c3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T12:01:13,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33013 {}] regionserver.HRegion(1722): Closing 940307e0d31686aca88e50f913c461c3, disabling compactions & flushes 2024-12-05T12:01:13,904 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33013 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733400072023,,1733400072057.940307e0d31686aca88e50f913c461c3. 2024-12-05T12:01:13,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33013 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733400072023,,1733400072057.940307e0d31686aca88e50f913c461c3. 2024-12-05T12:01:13,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33013 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733400072023,,1733400072057.940307e0d31686aca88e50f913c461c3. after waiting 0 ms 2024-12-05T12:01:13,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33013 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733400072023,,1733400072057.940307e0d31686aca88e50f913c461c3. 2024-12-05T12:01:13,906 DEBUG [StoreFileCloser-940307e0d31686aca88e50f913c461c3-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/default/testBlockEvictionOnRegionMove1733400072023/940307e0d31686aca88e50f913c461c3/family/95ced1f5aa3e4ecdb06d33025aa5f3fb 2024-12-05T12:01:13,909 WARN [hfile-prefetch-1733400072870 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/default/testBlockEvictionOnRegionMove1733400072023/940307e0d31686aca88e50f913c461c3/family/95ced1f5aa3e4ecdb06d33025aa5f3fb, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-12-05T12:01:13,910 DEBUG [hfile-prefetch-1733400072870 {}] hfile.PrefetchExecutor(121): Prefetch completed for 95ced1f5aa3e4ecdb06d33025aa5f3fb 2024-12-05T12:01:13,911 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33013 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733400072023,,1733400072057.940307e0d31686aca88e50f913c461c3. 2024-12-05T12:01:13,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33013 {}] regionserver.HRegion(1676): Region close journal for 940307e0d31686aca88e50f913c461c3: Waiting for close lock at 1733400073904Disabling compacts and flushes for region at 1733400073904Disabling writes for close at 1733400073904Writing region close event to WAL at 1733400073911 (+7 ms)Closed at 1733400073911 2024-12-05T12:01:13,926 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-05T12:01:13,926 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-05T12:01:13,926 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-05T12:01:13,926 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-05T12:01:13,944 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-05T12:01:13,944 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733400073926 2024-12-05T12:01:13,944 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-05T12:01:13,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41433 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-05T12:01:14,061 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 940307e0d31686aca88e50f913c461c3 2024-12-05T12:01:14,061 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-12-05T12:01:14,062 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 940307e0d31686aca88e50f913c461c3, disabling compactions & flushes 2024-12-05T12:01:14,062 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733400072023,,1733400072057.940307e0d31686aca88e50f913c461c3. 2024-12-05T12:01:14,062 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733400072023,,1733400072057.940307e0d31686aca88e50f913c461c3. 2024-12-05T12:01:14,062 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733400072023,,1733400072057.940307e0d31686aca88e50f913c461c3. after waiting 0 ms 2024-12-05T12:01:14,062 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733400072023,,1733400072057.940307e0d31686aca88e50f913c461c3. 2024-12-05T12:01:14,063 DEBUG [StoreFileCloser-940307e0d31686aca88e50f913c461c3-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file 95ced1f5aa3e4ecdb06d33025aa5f3fb, starting offset: 0, end offset: 9223372036854775807 2024-12-05T12:01:14,064 DEBUG [StoreFileCloser-940307e0d31686aca88e50f913c461c3-family-1 {}] bucket.BucketCache(856): removed key 95ced1f5aa3e4ecdb06d33025aa5f3fb_0 from back map with offset lock 2094080 in the evict process 2024-12-05T12:01:14,069 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/default/testBlockEvictionOnRegionMove1733400072023/940307e0d31686aca88e50f913c461c3/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-05T12:01:14,069 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-05T12:01:14,069 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-05T12:01:14,070 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-05T12:01:14,070 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-05T12:01:14,070 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733400072023,,1733400072057.940307e0d31686aca88e50f913c461c3. 2024-12-05T12:01:14,070 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 940307e0d31686aca88e50f913c461c3: Waiting for close lock at 1733400074062Running coprocessor pre-close hooks at 1733400074062Disabling compacts and flushes for region at 1733400074062Disabling writes for close at 1733400074062Writing region close event to WAL at 1733400074065 (+3 ms)Running coprocessor post-close hooks at 1733400074069 (+4 ms)Closed at 1733400074070 (+1 ms) 2024-12-05T12:01:14,071 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 940307e0d31686aca88e50f913c461c3 move to 80666a11a09f,33013,1733400068967 record at close sequenceid=6 2024-12-05T12:01:14,074 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 940307e0d31686aca88e50f913c461c3 2024-12-05T12:01:14,074 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=940307e0d31686aca88e50f913c461c3, regionState=CLOSED 2024-12-05T12:01:14,076 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-05T12:01:14,076 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733400074070 2024-12-05T12:01:14,076 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-05T12:01:14,077 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 940307e0d31686aca88e50f913c461c3, server=80666a11a09f,34899,1733400068760 because future has completed 2024-12-05T12:01:14,081 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-05T12:01:14,081 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 940307e0d31686aca88e50f913c461c3, server=80666a11a09f,34899,1733400068760 in 175 msec 2024-12-05T12:01:14,082 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733400072023, region=940307e0d31686aca88e50f913c461c3, REOPEN/MOVE; state=CLOSED, location=80666a11a09f,33013,1733400068967; forceNewPlan=false, retain=false 2024-12-05T12:01:14,232 INFO [80666a11a09f:41433 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-05T12:01:14,233 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=940307e0d31686aca88e50f913c461c3, regionState=OPENING, regionLocation=80666a11a09f,33013,1733400068967 2024-12-05T12:01:14,236 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733400072023, region=940307e0d31686aca88e50f913c461c3, REOPEN/MOVE because future has completed 2024-12-05T12:01:14,237 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 940307e0d31686aca88e50f913c461c3, server=80666a11a09f,33013,1733400068967}] 2024-12-05T12:01:14,392 INFO [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1733400072023,,1733400072057.940307e0d31686aca88e50f913c461c3. 2024-12-05T12:01:14,392 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 940307e0d31686aca88e50f913c461c3, NAME => 'testBlockEvictionOnRegionMove1733400072023,,1733400072057.940307e0d31686aca88e50f913c461c3.', STARTKEY => '', ENDKEY => ''} 2024-12-05T12:01:14,393 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1733400072023 940307e0d31686aca88e50f913c461c3 2024-12-05T12:01:14,393 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733400072023,,1733400072057.940307e0d31686aca88e50f913c461c3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T12:01:14,393 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 940307e0d31686aca88e50f913c461c3 2024-12-05T12:01:14,393 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 940307e0d31686aca88e50f913c461c3 2024-12-05T12:01:14,394 INFO [StoreOpener-940307e0d31686aca88e50f913c461c3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 940307e0d31686aca88e50f913c461c3 2024-12-05T12:01:14,395 INFO [StoreOpener-940307e0d31686aca88e50f913c461c3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 940307e0d31686aca88e50f913c461c3 columnFamilyName family 2024-12-05T12:01:14,395 DEBUG [StoreOpener-940307e0d31686aca88e50f913c461c3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:01:14,397 DEBUG [StoreFileOpener-940307e0d31686aca88e50f913c461c3-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/default/testBlockEvictionOnRegionMove1733400072023/940307e0d31686aca88e50f913c461c3/family/95ced1f5aa3e4ecdb06d33025aa5f3fb, delay=925 ms 2024-12-05T12:01:14,398 DEBUG [StoreOpener-940307e0d31686aca88e50f913c461c3-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/default/testBlockEvictionOnRegionMove1733400072023/940307e0d31686aca88e50f913c461c3/family/95ced1f5aa3e4ecdb06d33025aa5f3fb 2024-12-05T12:01:14,399 INFO [StoreOpener-940307e0d31686aca88e50f913c461c3-1 {}] regionserver.HStore(327): Store=940307e0d31686aca88e50f913c461c3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T12:01:14,399 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 940307e0d31686aca88e50f913c461c3 2024-12-05T12:01:14,400 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/default/testBlockEvictionOnRegionMove1733400072023/940307e0d31686aca88e50f913c461c3 2024-12-05T12:01:14,401 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/default/testBlockEvictionOnRegionMove1733400072023/940307e0d31686aca88e50f913c461c3 2024-12-05T12:01:14,401 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 940307e0d31686aca88e50f913c461c3 2024-12-05T12:01:14,401 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 940307e0d31686aca88e50f913c461c3 2024-12-05T12:01:14,406 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 940307e0d31686aca88e50f913c461c3 2024-12-05T12:01:14,406 INFO [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 940307e0d31686aca88e50f913c461c3; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66408811, jitterRate=-0.010431602597236633}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T12:01:14,407 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 940307e0d31686aca88e50f913c461c3 2024-12-05T12:01:14,407 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 940307e0d31686aca88e50f913c461c3: Running coprocessor pre-open hook at 1733400074393Writing region info on filesystem at 1733400074393Initializing all the Stores at 1733400074394 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733400074394Cleaning up temporary data from old regions at 1733400074401 (+7 ms)Running coprocessor post-open hooks at 1733400074407 (+6 ms)Region opened successfully at 1733400074407 2024-12-05T12:01:14,408 INFO [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1733400072023,,1733400072057.940307e0d31686aca88e50f913c461c3., pid=9, masterSystemTime=1733400074389 2024-12-05T12:01:14,411 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1733400072023,,1733400072057.940307e0d31686aca88e50f913c461c3. 2024-12-05T12:01:14,411 INFO [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1733400072023,,1733400072057.940307e0d31686aca88e50f913c461c3. 2024-12-05T12:01:14,412 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=940307e0d31686aca88e50f913c461c3, regionState=OPEN, openSeqNum=10, regionLocation=80666a11a09f,33013,1733400068967 2024-12-05T12:01:14,415 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 940307e0d31686aca88e50f913c461c3, server=80666a11a09f,33013,1733400068967 because future has completed 2024-12-05T12:01:14,418 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-05T12:01:14,418 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 940307e0d31686aca88e50f913c461c3, server=80666a11a09f,33013,1733400068967 in 179 msec 2024-12-05T12:01:14,420 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733400072023, region=940307e0d31686aca88e50f913c461c3, REOPEN/MOVE in 525 msec 2024-12-05T12:01:14,450 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-05T12:01:14,450 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T12:01:14,450 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T12:01:14,455 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:01:14,456 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:01:14,456 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:01:14,457 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-05T12:01:14,457 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1058988911, stopped=false 2024-12-05T12:01:14,457 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=80666a11a09f,41433,1733400067757 2024-12-05T12:01:14,503 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34899-0x101a6a79fd00001, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T12:01:14,503 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41433-0x101a6a79fd00000, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T12:01:14,503 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33013-0x101a6a79fd00002, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T12:01:14,503 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34899-0x101a6a79fd00001, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:01:14,503 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41433-0x101a6a79fd00000, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:01:14,503 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33013-0x101a6a79fd00002, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:01:14,504 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T12:01:14,504 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T12:01:14,504 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T12:01:14,504 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:01:14,505 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '80666a11a09f,34899,1733400068760' ***** 2024-12-05T12:01:14,505 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-05T12:01:14,505 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '80666a11a09f,33013,1733400068967' ***** 2024-12-05T12:01:14,505 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-05T12:01:14,506 INFO [RS:0;80666a11a09f:34899 {}] bucket.BucketCache(1738): Disabling cache 2024-12-05T12:01:14,507 INFO [RS:0;80666a11a09f:34899 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/bucket.persistence 2024-12-05T12:01:14,507 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-05T12:01:14,507 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-05T12:01:14,507 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-05T12:01:14,507 INFO [RS:0;80666a11a09f:34899 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-05T12:01:14,507 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-05T12:01:14,508 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33013-0x101a6a79fd00002, quorum=127.0.0.1:51837, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T12:01:14,508 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34899-0x101a6a79fd00001, quorum=127.0.0.1:51837, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T12:01:14,508 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41433-0x101a6a79fd00000, quorum=127.0.0.1:51837, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T12:01:14,509 INFO [RS:1;80666a11a09f:33013 {}] bucket.BucketCache(1738): Disabling cache 2024-12-05T12:01:14,509 INFO [RS:1;80666a11a09f:33013 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/bucket.persistence 2024-12-05T12:01:14,509 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-05T12:01:14,509 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-05T12:01:14,509 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-05T12:01:14,509 INFO [RS:1;80666a11a09f:33013 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-05T12:01:14,509 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-05T12:01:14,517 DEBUG [RS:0;80666a11a09f:34899 {}] bucket.BucketCache(1487): Thread RS:0;80666a11a09f:34899 started persisting bucket cache to file 2024-12-05T12:01:14,517 DEBUG [RS:0;80666a11a09f:34899 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-05T12:01:14,518 DEBUG [RS:0;80666a11a09f:34899 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-05T12:01:14,519 DEBUG [RS:1;80666a11a09f:33013 {}] bucket.BucketCache(1487): Thread RS:1;80666a11a09f:33013 started persisting bucket cache to file 2024-12-05T12:01:14,520 DEBUG [RS:1;80666a11a09f:33013 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-05T12:01:14,520 DEBUG [RS:1;80666a11a09f:33013 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-05T12:01:14,522 DEBUG [RS:0;80666a11a09f:34899 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-05T12:01:14,522 DEBUG [RS:0;80666a11a09f:34899 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733400074517 2024-12-05T12:01:14,522 DEBUG [RS:0;80666a11a09f:34899 {}] bucket.BucketCache(1509): Thread RS:0;80666a11a09f:34899 finished persisting bucket cache to file, renaming 2024-12-05T12:01:14,523 INFO [RS:0;80666a11a09f:34899 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-05T12:01:14,523 DEBUG [RS:1;80666a11a09f:33013 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-05T12:01:14,523 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-05T12:01:14,523 INFO [RS:0;80666a11a09f:34899 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-05T12:01:14,523 DEBUG [RS:1;80666a11a09f:33013 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733400074519 2024-12-05T12:01:14,523 DEBUG [RS:1;80666a11a09f:33013 {}] bucket.BucketCache(1509): Thread RS:1;80666a11a09f:33013 finished persisting bucket cache to file, renaming 2024-12-05T12:01:14,523 INFO [RS:0;80666a11a09f:34899 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-05T12:01:14,523 INFO [RS:0;80666a11a09f:34899 {}] regionserver.HRegionServer(959): stopping server 80666a11a09f,34899,1733400068760 2024-12-05T12:01:14,523 INFO [RS:1;80666a11a09f:33013 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-05T12:01:14,523 INFO [RS:0;80666a11a09f:34899 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-05T12:01:14,524 INFO [RS:1;80666a11a09f:33013 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-05T12:01:14,524 INFO [RS:0;80666a11a09f:34899 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;80666a11a09f:34899. 2024-12-05T12:01:14,524 INFO [RS:1;80666a11a09f:33013 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-05T12:01:14,524 DEBUG [RS:0;80666a11a09f:34899 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T12:01:14,524 INFO [RS:1;80666a11a09f:33013 {}] regionserver.HRegionServer(3091): Received CLOSE for 940307e0d31686aca88e50f913c461c3 2024-12-05T12:01:14,524 DEBUG [RS:0;80666a11a09f:34899 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:01:14,524 INFO [RS:0;80666a11a09f:34899 {}] regionserver.HRegionServer(976): stopping server 80666a11a09f,34899,1733400068760; all regions closed. 2024-12-05T12:01:14,524 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-05T12:01:14,525 INFO [RS:1;80666a11a09f:33013 {}] regionserver.HRegionServer(959): stopping server 80666a11a09f,33013,1733400068967 2024-12-05T12:01:14,525 INFO [RS:1;80666a11a09f:33013 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-05T12:01:14,525 INFO [RS:1;80666a11a09f:33013 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;80666a11a09f:33013. 2024-12-05T12:01:14,525 DEBUG [RS:1;80666a11a09f:33013 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T12:01:14,525 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1733400072023,,1733400072057.940307e0d31686aca88e50f913c461c3. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-05T12:01:14,525 DEBUG [RS:1;80666a11a09f:33013 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:01:14,526 INFO [RS:1;80666a11a09f:33013 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-05T12:01:14,526 INFO [RS:1;80666a11a09f:33013 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-05T12:01:14,526 INFO [RS:1;80666a11a09f:33013 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-05T12:01:14,526 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 940307e0d31686aca88e50f913c461c3, disabling compactions & flushes 2024-12-05T12:01:14,526 INFO [RS:1;80666a11a09f:33013 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-05T12:01:14,526 INFO [RS:1;80666a11a09f:33013 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-05T12:01:14,526 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733400072023,,1733400072057.940307e0d31686aca88e50f913c461c3. 2024-12-05T12:01:14,527 DEBUG [RS:1;80666a11a09f:33013 {}] regionserver.HRegionServer(1325): Online Regions={940307e0d31686aca88e50f913c461c3=testBlockEvictionOnRegionMove1733400072023,,1733400072057.940307e0d31686aca88e50f913c461c3., 1588230740=hbase:meta,,1.1588230740} 2024-12-05T12:01:14,527 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733400072023,,1733400072057.940307e0d31686aca88e50f913c461c3. 2024-12-05T12:01:14,527 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733400072023,,1733400072057.940307e0d31686aca88e50f913c461c3. after waiting 0 ms 2024-12-05T12:01:14,527 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733400072023,,1733400072057.940307e0d31686aca88e50f913c461c3. 2024-12-05T12:01:14,527 DEBUG [RS:1;80666a11a09f:33013 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 940307e0d31686aca88e50f913c461c3 2024-12-05T12:01:14,527 INFO [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-05T12:01:14,527 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-05T12:01:14,527 INFO [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-05T12:01:14,527 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-05T12:01:14,528 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T12:01:14,528 DEBUG [StoreFileCloser-940307e0d31686aca88e50f913c461c3-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/default/testBlockEvictionOnRegionMove1733400072023/940307e0d31686aca88e50f913c461c3/family/95ced1f5aa3e4ecdb06d33025aa5f3fb 2024-12-05T12:01:14,528 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T12:01:14,534 INFO [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-12-05T12:01:14,538 DEBUG [RS:0;80666a11a09f:34899 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/oldWALs 2024-12-05T12:01:14,538 INFO [RS:0;80666a11a09f:34899 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 80666a11a09f%2C34899%2C1733400068760:(num 1733400071335) 2024-12-05T12:01:14,538 DEBUG [RS:0;80666a11a09f:34899 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:01:14,538 INFO [RS:0;80666a11a09f:34899 {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T12:01:14,539 INFO [RS:0;80666a11a09f:34899 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-05T12:01:14,540 INFO [RS:0;80666a11a09f:34899 {}] hbase.ChoreService(370): Chore service for: regionserver/80666a11a09f:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-05T12:01:14,540 INFO [RS:0;80666a11a09f:34899 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-05T12:01:14,540 INFO [RS:0;80666a11a09f:34899 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-05T12:01:14,540 INFO [regionserver/80666a11a09f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-05T12:01:14,540 INFO [RS:0;80666a11a09f:34899 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-05T12:01:14,540 INFO [RS:0;80666a11a09f:34899 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-05T12:01:14,541 INFO [RS:0;80666a11a09f:34899 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34899 2024-12-05T12:01:14,542 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/default/testBlockEvictionOnRegionMove1733400072023/940307e0d31686aca88e50f913c461c3/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-12-05T12:01:14,543 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733400072023,,1733400072057.940307e0d31686aca88e50f913c461c3. 2024-12-05T12:01:14,543 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 940307e0d31686aca88e50f913c461c3: Waiting for close lock at 1733400074525Running coprocessor pre-close hooks at 1733400074526 (+1 ms)Disabling compacts and flushes for region at 1733400074526Disabling writes for close at 1733400074527 (+1 ms)Writing region close event to WAL at 1733400074528 (+1 ms)Running coprocessor post-close hooks at 1733400074543 (+15 ms)Closed at 1733400074543 2024-12-05T12:01:14,543 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1733400072023,,1733400072057.940307e0d31686aca88e50f913c461c3. 2024-12-05T12:01:14,553 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 1acbf0a871864e428d031409454be275 2024-12-05T12:01:14,554 INFO [RS:0;80666a11a09f:34899 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-05T12:01:14,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34899-0x101a6a79fd00001, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/80666a11a09f,34899,1733400068760 2024-12-05T12:01:14,555 ERROR [Time-limited test-EventThread {}] zookeeper.ClientCnxn$EventThread(581): Error while calling watcher. java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.trace.TraceUtil$$Lambda$249/0x00007fe6f847caf8@4f9674cc rejected from java.util.concurrent.ThreadPoolExecutor@a94f35a[Shutting down, pool size = 1, active threads = 0, queued tasks = 0, completed tasks = 14] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1360) ~[?:?] at java.util.concurrent.Executors$DelegatedExecutorService.execute(Executors.java:721) ~[?:?] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.process(ZKWatcher.java:613) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:579) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:554) ~[zookeeper-3.8.4.jar:3.8.4] 2024-12-05T12:01:14,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41433-0x101a6a79fd00000, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T12:01:14,557 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [80666a11a09f,34899,1733400068760] 2024-12-05T12:01:14,571 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/80666a11a09f,34899,1733400068760 already deleted, retry=false 2024-12-05T12:01:14,571 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 80666a11a09f,34899,1733400068760 expired; onlineServers=1 2024-12-05T12:01:14,572 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/.tmp/info/1acbf0a871864e428d031409454be275 is 205, key is testBlockEvictionOnRegionMove1733400072023,,1733400072057.940307e0d31686aca88e50f913c461c3./info:regioninfo/1733400074411/Put/seqid=0 2024-12-05T12:01:14,573 INFO [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/.tmp/info/1acbf0a871864e428d031409454be275 2024-12-05T12:01:14,577 INFO [regionserver/80666a11a09f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T12:01:14,582 INFO [regionserver/80666a11a09f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T12:01:14,593 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on fd4cbab4a19d4fc69ef1c68f75c1abd7 2024-12-05T12:01:14,612 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/.tmp/ns/fd4cbab4a19d4fc69ef1c68f75c1abd7 is 43, key is default/ns:d/1733400071832/Put/seqid=0 2024-12-05T12:01:14,613 INFO [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/.tmp/ns/fd4cbab4a19d4fc69ef1c68f75c1abd7 2024-12-05T12:01:14,626 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 1417300a0ab24de08f67c75a1cdd86e6 2024-12-05T12:01:14,644 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/.tmp/table/1417300a0ab24de08f67c75a1cdd86e6 is 78, key is testBlockEvictionOnRegionMove1733400072023/table:state/1733400072520/Put/seqid=0 2024-12-05T12:01:14,645 INFO [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/.tmp/table/1417300a0ab24de08f67c75a1cdd86e6 2024-12-05T12:01:14,648 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/.tmp/info/1acbf0a871864e428d031409454be275 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/info/1acbf0a871864e428d031409454be275 2024-12-05T12:01:14,650 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/info/1acbf0a871864e428d031409454be275, delay=920 ms 2024-12-05T12:01:14,651 INFO [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/info/1acbf0a871864e428d031409454be275, entries=15, sequenceid=15, filesize=7.6 K 2024-12-05T12:01:14,652 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/.tmp/ns/fd4cbab4a19d4fc69ef1c68f75c1abd7 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/ns/fd4cbab4a19d4fc69ef1c68f75c1abd7 2024-12-05T12:01:14,654 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/ns/fd4cbab4a19d4fc69ef1c68f75c1abd7, delay=985 ms 2024-12-05T12:01:14,655 INFO [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/ns/fd4cbab4a19d4fc69ef1c68f75c1abd7, entries=2, sequenceid=15, filesize=5.0 K 2024-12-05T12:01:14,656 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/.tmp/table/1417300a0ab24de08f67c75a1cdd86e6 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/table/1417300a0ab24de08f67c75a1cdd86e6 2024-12-05T12:01:14,658 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/table/1417300a0ab24de08f67c75a1cdd86e6, delay=917 ms 2024-12-05T12:01:14,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34899-0x101a6a79fd00001, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T12:01:14,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34899-0x101a6a79fd00001, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T12:01:14,664 INFO [RS:0;80666a11a09f:34899 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-05T12:01:14,664 INFO [RS:0;80666a11a09f:34899 {}] regionserver.HRegionServer(1031): Exiting; stopping=80666a11a09f,34899,1733400068760; zookeeper connection closed. 2024-12-05T12:01:14,664 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@e50c4c5 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@e50c4c5 2024-12-05T12:01:14,667 INFO [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/table/1417300a0ab24de08f67c75a1cdd86e6, entries=2, sequenceid=15, filesize=5.3 K 2024-12-05T12:01:14,667 INFO [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 139ms, sequenceid=15, compaction requested=false 2024-12-05T12:01:14,667 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-05T12:01:14,669 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/info/1acbf0a871864e428d031409454be275 2024-12-05T12:01:14,669 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/ns/fd4cbab4a19d4fc69ef1c68f75c1abd7 2024-12-05T12:01:14,670 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/table/1417300a0ab24de08f67c75a1cdd86e6 2024-12-05T12:01:14,675 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-12-05T12:01:14,675 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-05T12:01:14,676 INFO [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-05T12:01:14,676 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733400074527Running coprocessor pre-close hooks at 1733400074527Disabling compacts and flushes for region at 1733400074527Disabling writes for close at 1733400074528 (+1 ms)Obtaining lock to block concurrent updates at 1733400074535 (+7 ms)Preparing flush snapshotting stores in 1588230740 at 1733400074535Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1733400074535Flushing stores of hbase:meta,,1.1588230740 at 1733400074537 (+2 ms)Flushing 1588230740/info: creating writer at 1733400074539 (+2 ms)Flushing 1588230740/info: appending metadata at 1733400074570 (+31 ms)Flushing 1588230740/info: closing flushed file at 1733400074571 (+1 ms)Flushing 1588230740/ns: creating writer at 1733400074577 (+6 ms)Flushing 1588230740/ns: appending metadata at 1733400074611 (+34 ms)Flushing 1588230740/ns: closing flushed file at 1733400074611Flushing 1588230740/table: creating writer at 1733400074615 (+4 ms)Flushing 1588230740/table: appending metadata at 1733400074643 (+28 ms)Flushing 1588230740/table: closing flushed file at 1733400074643Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2afc14e0: reopening flushed file at 1733400074648 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@20cb80de: reopening flushed file at 1733400074652 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@57ab72c2: reopening flushed file at 1733400074656 (+4 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 139ms, sequenceid=15, compaction requested=false at 1733400074667 (+11 ms)Writing region close event to WAL at 1733400074670 (+3 ms)Running coprocessor post-close hooks at 1733400074675 (+5 ms)Closed at 1733400074676 (+1 ms) 2024-12-05T12:01:14,676 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-05T12:01:14,727 INFO [RS:1;80666a11a09f:33013 {}] regionserver.HRegionServer(976): stopping server 80666a11a09f,33013,1733400068967; all regions closed. 2024-12-05T12:01:14,730 DEBUG [RS:1;80666a11a09f:33013 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/oldWALs 2024-12-05T12:01:14,730 INFO [RS:1;80666a11a09f:33013 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 80666a11a09f%2C33013%2C1733400068967.meta:.meta(num 1733400071640) 2024-12-05T12:01:14,733 DEBUG [RS:1;80666a11a09f:33013 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/oldWALs 2024-12-05T12:01:14,733 INFO [RS:1;80666a11a09f:33013 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 80666a11a09f%2C33013%2C1733400068967:(num 1733400071335) 2024-12-05T12:01:14,733 DEBUG [RS:1;80666a11a09f:33013 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:01:14,733 INFO [RS:1;80666a11a09f:33013 {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T12:01:14,733 INFO [RS:1;80666a11a09f:33013 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-05T12:01:14,733 INFO [RS:1;80666a11a09f:33013 {}] hbase.ChoreService(370): Chore service for: regionserver/80666a11a09f:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-05T12:01:14,734 INFO [RS:1;80666a11a09f:33013 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-05T12:01:14,734 INFO [regionserver/80666a11a09f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-05T12:01:14,734 INFO [RS:1;80666a11a09f:33013 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33013 2024-12-05T12:01:14,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33013-0x101a6a79fd00002, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/80666a11a09f,33013,1733400068967 2024-12-05T12:01:14,745 INFO [RS:1;80666a11a09f:33013 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-05T12:01:14,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41433-0x101a6a79fd00000, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T12:01:14,755 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [80666a11a09f,33013,1733400068967] 2024-12-05T12:01:14,763 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/80666a11a09f,33013,1733400068967 already deleted, retry=false 2024-12-05T12:01:14,763 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 80666a11a09f,33013,1733400068967 expired; onlineServers=0 2024-12-05T12:01:14,763 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '80666a11a09f,41433,1733400067757' ***** 2024-12-05T12:01:14,763 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-05T12:01:14,763 INFO [M:0;80666a11a09f:41433 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-05T12:01:14,763 INFO [M:0;80666a11a09f:41433 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-05T12:01:14,764 DEBUG [M:0;80666a11a09f:41433 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-05T12:01:14,764 DEBUG [M:0;80666a11a09f:41433 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-05T12:01:14,764 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-05T12:01:14,764 DEBUG [master/80666a11a09f:0:becomeActiveMaster-HFileCleaner.large.0-1733400070636 {}] cleaner.HFileCleaner(306): Exit Thread[master/80666a11a09f:0:becomeActiveMaster-HFileCleaner.large.0-1733400070636,5,FailOnTimeoutGroup] 2024-12-05T12:01:14,764 DEBUG [master/80666a11a09f:0:becomeActiveMaster-HFileCleaner.small.0-1733400070639 {}] cleaner.HFileCleaner(306): Exit Thread[master/80666a11a09f:0:becomeActiveMaster-HFileCleaner.small.0-1733400070639,5,FailOnTimeoutGroup] 2024-12-05T12:01:14,764 INFO [M:0;80666a11a09f:41433 {}] hbase.ChoreService(370): Chore service for: master/80666a11a09f:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-05T12:01:14,764 INFO [M:0;80666a11a09f:41433 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-05T12:01:14,764 DEBUG [M:0;80666a11a09f:41433 {}] master.HMaster(1795): Stopping service threads 2024-12-05T12:01:14,764 INFO [M:0;80666a11a09f:41433 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-05T12:01:14,765 INFO [M:0;80666a11a09f:41433 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T12:01:14,765 INFO [M:0;80666a11a09f:41433 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T12:01:14,765 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-05T12:01:14,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41433-0x101a6a79fd00000, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-05T12:01:14,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41433-0x101a6a79fd00000, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:01:14,772 DEBUG [M:0;80666a11a09f:41433 {}] zookeeper.ZKUtil(347): master:41433-0x101a6a79fd00000, quorum=127.0.0.1:51837, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-05T12:01:14,772 WARN [M:0;80666a11a09f:41433 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-05T12:01:14,772 INFO [M:0;80666a11a09f:41433 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/.lastflushedseqids 2024-12-05T12:01:14,796 INFO [M:0;80666a11a09f:41433 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T12:01:14,796 INFO [M:0;80666a11a09f:41433 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-05T12:01:14,797 DEBUG [M:0;80666a11a09f:41433 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-05T12:01:14,797 INFO [M:0;80666a11a09f:41433 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T12:01:14,797 DEBUG [M:0;80666a11a09f:41433 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T12:01:14,797 DEBUG [M:0;80666a11a09f:41433 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-05T12:01:14,797 DEBUG [M:0;80666a11a09f:41433 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T12:01:14,797 INFO [M:0;80666a11a09f:41433 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.67 KB heapSize=43.02 KB 2024-12-05T12:01:14,807 DEBUG [M:0;80666a11a09f:41433 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 5636afdb00344321847952f58444a2ac 2024-12-05T12:01:14,821 DEBUG [M:0;80666a11a09f:41433 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5636afdb00344321847952f58444a2ac is 82, key is hbase:meta,,1/info:regioninfo/1733400071715/Put/seqid=0 2024-12-05T12:01:14,821 INFO [M:0;80666a11a09f:41433 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5636afdb00344321847952f58444a2ac 2024-12-05T12:01:14,832 DEBUG [M:0;80666a11a09f:41433 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b91a2b7598f5485cbbdb2ab3852a2c06 2024-12-05T12:01:14,850 DEBUG [M:0;80666a11a09f:41433 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b91a2b7598f5485cbbdb2ab3852a2c06 is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733400072528/Put/seqid=0 2024-12-05T12:01:14,853 INFO [M:0;80666a11a09f:41433 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.05 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b91a2b7598f5485cbbdb2ab3852a2c06 2024-12-05T12:01:14,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33013-0x101a6a79fd00002, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T12:01:14,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33013-0x101a6a79fd00002, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T12:01:14,857 INFO [RS:1;80666a11a09f:33013 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-05T12:01:14,857 INFO [RS:1;80666a11a09f:33013 {}] regionserver.HRegionServer(1031): Exiting; stopping=80666a11a09f,33013,1733400068967; zookeeper connection closed. 2024-12-05T12:01:14,858 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@31ce139c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@31ce139c 2024-12-05T12:01:14,859 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-12-05T12:01:14,864 DEBUG [M:0;80666a11a09f:41433 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f6cf7e970e5148449d2aa9ec035c92aa 2024-12-05T12:01:14,877 DEBUG [M:0;80666a11a09f:41433 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f6cf7e970e5148449d2aa9ec035c92aa is 69, key is 80666a11a09f,33013,1733400068967/rs:state/1733400070745/Put/seqid=0 2024-12-05T12:01:14,878 INFO [M:0;80666a11a09f:41433 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f6cf7e970e5148449d2aa9ec035c92aa 2024-12-05T12:01:14,880 DEBUG [M:0;80666a11a09f:41433 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5636afdb00344321847952f58444a2ac as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5636afdb00344321847952f58444a2ac 2024-12-05T12:01:14,882 INFO [M:0;80666a11a09f:41433 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5636afdb00344321847952f58444a2ac, entries=8, sequenceid=84, filesize=5.5 K 2024-12-05T12:01:14,883 DEBUG [M:0;80666a11a09f:41433 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b91a2b7598f5485cbbdb2ab3852a2c06 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b91a2b7598f5485cbbdb2ab3852a2c06 2024-12-05T12:01:14,885 INFO [M:0;80666a11a09f:41433 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b91a2b7598f5485cbbdb2ab3852a2c06, entries=9, sequenceid=84, filesize=6.5 K 2024-12-05T12:01:14,886 DEBUG [M:0;80666a11a09f:41433 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f6cf7e970e5148449d2aa9ec035c92aa as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f6cf7e970e5148449d2aa9ec035c92aa 2024-12-05T12:01:14,888 INFO [M:0;80666a11a09f:41433 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f6cf7e970e5148449d2aa9ec035c92aa, entries=2, sequenceid=84, filesize=5.1 K 2024-12-05T12:01:14,889 INFO [M:0;80666a11a09f:41433 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 92ms, sequenceid=84, compaction requested=false 2024-12-05T12:01:14,909 INFO [M:0;80666a11a09f:41433 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T12:01:14,909 DEBUG [M:0;80666a11a09f:41433 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733400074797Disabling compacts and flushes for region at 1733400074797Disabling writes for close at 1733400074797Obtaining lock to block concurrent updates at 1733400074797Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733400074797Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35506, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1733400074798 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733400074798Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733400074798Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733400074820 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733400074820Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733400074824 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733400074848 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733400074848Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733400074855 (+7 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733400074877 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733400074877Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6735c6f7: reopening flushed file at 1733400074880 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@47dd1205: reopening flushed file at 1733400074882 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b906bfd: reopening flushed file at 1733400074885 (+3 ms)Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 92ms, sequenceid=84, compaction requested=false at 1733400074889 (+4 ms)Writing region close event to WAL at 1733400074909 (+20 ms)Closed at 1733400074909 2024-12-05T12:01:14,911 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-05T12:01:14,911 INFO [M:0;80666a11a09f:41433 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-05T12:01:14,911 INFO [M:0;80666a11a09f:41433 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41433 2024-12-05T12:01:14,912 INFO [M:0;80666a11a09f:41433 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-05T12:01:15,022 INFO [M:0;80666a11a09f:41433 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-05T12:01:15,022 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41433-0x101a6a79fd00000, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T12:01:15,022 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41433-0x101a6a79fd00000, quorum=127.0.0.1:51837, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T12:01:15,028 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-05T12:01:15,052 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-05T12:01:15,052 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T12:01:15,063 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=51 (was 12) Potentially hanging thread: hfile-prefetch-1733400074654 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733400074650 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733400072870 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733400074397 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/80666a11a09f:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: master/80666a11a09f:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/80666a11a09f:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/80666a11a09f:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1016 (was 1052), ProcessCount=11 (was 11), AvailableMemoryMB=7381 (was 7747) 2024-12-05T12:01:15,069 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=51, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=1016, ProcessCount=11, AvailableMemoryMB=7381 2024-12-05T12:01:15,069 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/hadoop-log-dir so I do NOT create it in target/test-data/4716cd85-2b6b-3a48-c1bf-78df3670c43e 2024-12-05T12:01:15,070 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/hadoop-tmp-dir so I do NOT create it in target/test-data/4716cd85-2b6b-3a48-c1bf-78df3670c43e 2024-12-05T12:01:15,070 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T12:01:15,072 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4716cd85-2b6b-3a48-c1bf-78df3670c43e/cluster_be478135-36d1-600a-e65e-ba144c69393d, deleteOnExit=true 2024-12-05T12:01:15,093 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4716cd85-2b6b-3a48-c1bf-78df3670c43e/cluster_be478135-36d1-600a-e65e-ba144c69393d/zookeeper_0, clientPort=63500, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4716cd85-2b6b-3a48-c1bf-78df3670c43e/cluster_be478135-36d1-600a-e65e-ba144c69393d/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4716cd85-2b6b-3a48-c1bf-78df3670c43e/cluster_be478135-36d1-600a-e65e-ba144c69393d/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-05T12:01:15,094 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63500 2024-12-05T12:01:15,108 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6 with version=8 2024-12-05T12:01:15,108 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/hbase-staging 2024-12-05T12:01:15,110 INFO [Time-limited test {}] client.ConnectionUtils(128): master/80666a11a09f:0 server-side Connection retries=45 2024-12-05T12:01:15,110 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T12:01:15,111 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T12:01:15,111 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T12:01:15,111 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T12:01:15,111 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T12:01:15,111 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-05T12:01:15,111 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T12:01:15,112 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46679 2024-12-05T12:01:15,113 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:46679 connecting to ZooKeeper ensemble=127.0.0.1:63500 2024-12-05T12:01:15,150 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:466790x0, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T12:01:15,150 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46679-0x101a6a7bdc80000 connected 2024-12-05T12:01:15,218 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T12:01:15,218 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T12:01:15,219 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46679-0x101a6a7bdc80000, quorum=127.0.0.1:63500, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T12:01:15,219 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6, hbase.cluster.distributed=false 2024-12-05T12:01:15,221 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46679-0x101a6a7bdc80000, quorum=127.0.0.1:63500, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T12:01:15,222 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46679 2024-12-05T12:01:15,222 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46679 2024-12-05T12:01:15,226 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46679 2024-12-05T12:01:15,227 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46679 2024-12-05T12:01:15,227 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46679 2024-12-05T12:01:15,251 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/80666a11a09f:0 server-side Connection retries=45 2024-12-05T12:01:15,251 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T12:01:15,251 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T12:01:15,252 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T12:01:15,252 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T12:01:15,252 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T12:01:15,252 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-05T12:01:15,252 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T12:01:15,253 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40679 2024-12-05T12:01:15,255 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40679 connecting to ZooKeeper ensemble=127.0.0.1:63500 2024-12-05T12:01:15,255 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T12:01:15,255 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T12:01:15,276 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:406790x0, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T12:01:15,277 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40679-0x101a6a7bdc80001 connected 2024-12-05T12:01:15,277 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:406790x0, quorum=127.0.0.1:63500, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T12:01:15,278 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-05T12:01:15,279 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4716cd85-2b6b-3a48-c1bf-78df3670c43e/bucket.cache 2024-12-05T12:01:15,279 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-05T12:01:15,279 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-05T12:01:15,279 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-05T12:01:15,281 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4716cd85-2b6b-3a48-c1bf-78df3670c43e/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4716cd85-2b6b-3a48-c1bf-78df3670c43e/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-05T12:01:15,281 INFO [Thread-119 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-05T12:01:15,281 WARN [Thread-119 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-05T12:01:15,281 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-05T12:01:15,281 INFO [Thread-119 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4716cd85-2b6b-3a48-c1bf-78df3670c43e/bucket.persistence is complete. 2024-12-05T12:01:15,282 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-05T12:01:15,287 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40679-0x101a6a7bdc80001, quorum=127.0.0.1:63500, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-05T12:01:15,288 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40679-0x101a6a7bdc80001, quorum=127.0.0.1:63500, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T12:01:15,294 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40679 2024-12-05T12:01:15,294 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40679 2024-12-05T12:01:15,299 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40679 2024-12-05T12:01:15,311 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40679 2024-12-05T12:01:15,311 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40679 2024-12-05T12:01:15,332 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/80666a11a09f:0 server-side Connection retries=45 2024-12-05T12:01:15,332 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T12:01:15,332 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T12:01:15,332 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T12:01:15,332 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T12:01:15,332 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T12:01:15,333 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-05T12:01:15,333 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T12:01:15,334 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39323 2024-12-05T12:01:15,335 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39323 connecting to ZooKeeper ensemble=127.0.0.1:63500 2024-12-05T12:01:15,336 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T12:01:15,336 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T12:01:15,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:393230x0, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T12:01:15,352 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:393230x0, quorum=127.0.0.1:63500, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T12:01:15,352 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-05T12:01:15,363 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39323-0x101a6a7bdc80002 connected 2024-12-05T12:01:15,363 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4716cd85-2b6b-3a48-c1bf-78df3670c43e/bucket.cache 2024-12-05T12:01:15,363 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-05T12:01:15,363 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-05T12:01:15,363 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-05T12:01:15,364 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-05T12:01:15,364 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4716cd85-2b6b-3a48-c1bf-78df3670c43e/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4716cd85-2b6b-3a48-c1bf-78df3670c43e/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-05T12:01:15,364 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-05T12:01:15,364 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-05T12:01:15,364 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4716cd85-2b6b-3a48-c1bf-78df3670c43e/bucket.persistence is complete. 2024-12-05T12:01:15,365 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-05T12:01:15,366 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39323-0x101a6a7bdc80002, quorum=127.0.0.1:63500, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-05T12:01:15,372 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39323-0x101a6a7bdc80002, quorum=127.0.0.1:63500, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T12:01:15,394 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39323 2024-12-05T12:01:15,395 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39323 2024-12-05T12:01:15,395 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39323 2024-12-05T12:01:15,396 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39323 2024-12-05T12:01:15,396 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39323 2024-12-05T12:01:15,414 DEBUG [M:0;80666a11a09f:46679 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;80666a11a09f:46679 2024-12-05T12:01:15,415 INFO [master/80666a11a09f:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/80666a11a09f,46679,1733400075110 2024-12-05T12:01:15,423 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46679-0x101a6a7bdc80000, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T12:01:15,423 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39323-0x101a6a7bdc80002, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T12:01:15,423 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40679-0x101a6a7bdc80001, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T12:01:15,425 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46679-0x101a6a7bdc80000, quorum=127.0.0.1:63500, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/80666a11a09f,46679,1733400075110 2024-12-05T12:01:15,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39323-0x101a6a7bdc80002, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-05T12:01:15,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40679-0x101a6a7bdc80001, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-05T12:01:15,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39323-0x101a6a7bdc80002, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:01:15,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40679-0x101a6a7bdc80001, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:01:15,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46679-0x101a6a7bdc80000, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:01:15,437 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46679-0x101a6a7bdc80000, quorum=127.0.0.1:63500, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-05T12:01:15,437 INFO [master/80666a11a09f:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/80666a11a09f,46679,1733400075110 from backup master directory 2024-12-05T12:01:15,448 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46679-0x101a6a7bdc80000, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/80666a11a09f,46679,1733400075110 2024-12-05T12:01:15,448 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40679-0x101a6a7bdc80001, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T12:01:15,448 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46679-0x101a6a7bdc80000, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T12:01:15,448 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39323-0x101a6a7bdc80002, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T12:01:15,449 WARN [master/80666a11a09f:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T12:01:15,449 INFO [master/80666a11a09f:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=80666a11a09f,46679,1733400075110 2024-12-05T12:01:15,450 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/hbase.id] with ID: 2e709e1b-2712-44bd-9a60-dab809f7a2fa 2024-12-05T12:01:15,451 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/.tmp/hbase.id 2024-12-05T12:01:15,454 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/hbase.id] 2024-12-05T12:01:15,487 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T12:01:15,487 INFO [master/80666a11a09f:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-05T12:01:15,489 INFO [master/80666a11a09f:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-12-05T12:01:15,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46679-0x101a6a7bdc80000, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:01:15,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40679-0x101a6a7bdc80001, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:01:15,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39323-0x101a6a7bdc80002, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:01:15,503 INFO [master/80666a11a09f:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T12:01:15,505 INFO [master/80666a11a09f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-05T12:01:15,505 INFO [master/80666a11a09f:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T12:01:15,525 INFO [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/MasterData/data/master/store 2024-12-05T12:01:15,531 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T12:01:15,532 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-05T12:01:15,532 INFO [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T12:01:15,532 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T12:01:15,532 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-05T12:01:15,532 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T12:01:15,532 INFO [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T12:01:15,532 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733400075532Disabling compacts and flushes for region at 1733400075532Disabling writes for close at 1733400075532Writing region close event to WAL at 1733400075532Closed at 1733400075532 2024-12-05T12:01:15,532 WARN [master/80666a11a09f:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/MasterData/data/master/store/.initializing 2024-12-05T12:01:15,532 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/MasterData/WALs/80666a11a09f,46679,1733400075110 2024-12-05T12:01:15,537 INFO [master/80666a11a09f:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T12:01:15,546 INFO [master/80666a11a09f:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=80666a11a09f%2C46679%2C1733400075110, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/MasterData/WALs/80666a11a09f,46679,1733400075110, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/MasterData/oldWALs, maxLogs=10 2024-12-05T12:01:15,563 INFO [master/80666a11a09f:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/MasterData/WALs/80666a11a09f,46679,1733400075110/80666a11a09f%2C46679%2C1733400075110.1733400075546 2024-12-05T12:01:15,563 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-05T12:01:15,564 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-05T12:01:15,564 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T12:01:15,564 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T12:01:15,564 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T12:01:15,565 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T12:01:15,574 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-05T12:01:15,574 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:01:15,575 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T12:01:15,575 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T12:01:15,583 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-05T12:01:15,584 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:01:15,584 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T12:01:15,584 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T12:01:15,603 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-05T12:01:15,603 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:01:15,604 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T12:01:15,604 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T12:01:15,608 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-05T12:01:15,608 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:01:15,608 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T12:01:15,608 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T12:01:15,609 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-05T12:01:15,609 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-05T12:01:15,609 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T12:01:15,610 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T12:01:15,610 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T12:01:15,610 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T12:01:15,616 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T12:01:15,617 INFO [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72315462, jitterRate=0.07758435606956482}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T12:01:15,618 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733400075564Initializing all the Stores at 1733400075565 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733400075565Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733400075565Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733400075565Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733400075565Cleaning up temporary data from old regions at 1733400075610 (+45 ms)Region opened successfully at 1733400075618 (+8 ms) 2024-12-05T12:01:15,618 INFO [master/80666a11a09f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-05T12:01:15,622 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4df210a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=80666a11a09f/172.17.0.2:0 2024-12-05T12:01:15,624 INFO [master/80666a11a09f:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-05T12:01:15,624 INFO [master/80666a11a09f:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-05T12:01:15,624 INFO [master/80666a11a09f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-05T12:01:15,624 INFO [master/80666a11a09f:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T12:01:15,625 INFO [master/80666a11a09f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-05T12:01:15,625 INFO [master/80666a11a09f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-12-05T12:01:15,625 INFO [master/80666a11a09f:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-05T12:01:15,627 INFO [master/80666a11a09f:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-05T12:01:15,628 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46679-0x101a6a7bdc80000, quorum=127.0.0.1:63500, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-05T12:01:15,634 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-05T12:01:15,635 INFO [master/80666a11a09f:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-05T12:01:15,636 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46679-0x101a6a7bdc80000, quorum=127.0.0.1:63500, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-05T12:01:15,643 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-05T12:01:15,644 INFO [master/80666a11a09f:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-05T12:01:15,645 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46679-0x101a6a7bdc80000, quorum=127.0.0.1:63500, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-05T12:01:15,651 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-05T12:01:15,652 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46679-0x101a6a7bdc80000, quorum=127.0.0.1:63500, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-05T12:01:15,659 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-05T12:01:15,662 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46679-0x101a6a7bdc80000, quorum=127.0.0.1:63500, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-05T12:01:15,667 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-05T12:01:15,676 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40679-0x101a6a7bdc80001, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T12:01:15,676 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39323-0x101a6a7bdc80002, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T12:01:15,676 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46679-0x101a6a7bdc80000, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T12:01:15,676 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40679-0x101a6a7bdc80001, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:01:15,676 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46679-0x101a6a7bdc80000, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:01:15,676 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39323-0x101a6a7bdc80002, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:01:15,677 INFO [master/80666a11a09f:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=80666a11a09f,46679,1733400075110, sessionid=0x101a6a7bdc80000, setting cluster-up flag (Was=false) 2024-12-05T12:01:15,693 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40679-0x101a6a7bdc80001, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:01:15,693 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39323-0x101a6a7bdc80002, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:01:15,693 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46679-0x101a6a7bdc80000, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:01:15,718 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-05T12:01:15,719 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=80666a11a09f,46679,1733400075110 2024-12-05T12:01:15,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39323-0x101a6a7bdc80002, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:01:15,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40679-0x101a6a7bdc80001, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:01:15,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46679-0x101a6a7bdc80000, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:01:15,759 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-05T12:01:15,761 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=80666a11a09f,46679,1733400075110 2024-12-05T12:01:15,761 INFO [master/80666a11a09f:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-05T12:01:15,763 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T12:01:15,763 INFO [master/80666a11a09f:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-05T12:01:15,763 INFO [master/80666a11a09f:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-05T12:01:15,763 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 80666a11a09f,46679,1733400075110 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-05T12:01:15,765 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/80666a11a09f:0, corePoolSize=5, maxPoolSize=5 2024-12-05T12:01:15,765 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/80666a11a09f:0, corePoolSize=5, maxPoolSize=5 2024-12-05T12:01:15,765 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/80666a11a09f:0, corePoolSize=5, maxPoolSize=5 2024-12-05T12:01:15,765 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/80666a11a09f:0, corePoolSize=5, maxPoolSize=5 2024-12-05T12:01:15,765 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/80666a11a09f:0, corePoolSize=10, maxPoolSize=10 2024-12-05T12:01:15,765 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:15,765 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/80666a11a09f:0, corePoolSize=2, maxPoolSize=2 2024-12-05T12:01:15,765 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:15,767 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T12:01:15,767 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-05T12:01:15,768 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:01:15,768 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-05T12:01:15,771 INFO [master/80666a11a09f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733400105770 2024-12-05T12:01:15,771 INFO [master/80666a11a09f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-05T12:01:15,771 INFO [master/80666a11a09f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-05T12:01:15,771 INFO [master/80666a11a09f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-05T12:01:15,771 INFO [master/80666a11a09f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-05T12:01:15,771 INFO [master/80666a11a09f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-05T12:01:15,771 INFO [master/80666a11a09f:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-05T12:01:15,774 INFO [master/80666a11a09f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:15,779 INFO [master/80666a11a09f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-05T12:01:15,779 INFO [master/80666a11a09f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-05T12:01:15,779 INFO [master/80666a11a09f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-05T12:01:15,787 INFO [master/80666a11a09f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-05T12:01:15,787 INFO [master/80666a11a09f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-05T12:01:15,790 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/80666a11a09f:0:becomeActiveMaster-HFileCleaner.large.0-1733400075787,5,FailOnTimeoutGroup] 2024-12-05T12:01:15,791 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/80666a11a09f:0:becomeActiveMaster-HFileCleaner.small.0-1733400075791,5,FailOnTimeoutGroup] 2024-12-05T12:01:15,791 INFO [master/80666a11a09f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:15,791 INFO [master/80666a11a09f:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-05T12:01:15,791 INFO [master/80666a11a09f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:15,791 INFO [master/80666a11a09f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:15,804 INFO [RS:0;80666a11a09f:40679 {}] regionserver.HRegionServer(746): ClusterId : 2e709e1b-2712-44bd-9a60-dab809f7a2fa 2024-12-05T12:01:15,804 DEBUG [RS:0;80666a11a09f:40679 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-05T12:01:15,813 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-05T12:01:15,813 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6 2024-12-05T12:01:15,815 INFO [RS:1;80666a11a09f:39323 {}] regionserver.HRegionServer(746): ClusterId : 2e709e1b-2712-44bd-9a60-dab809f7a2fa 2024-12-05T12:01:15,815 DEBUG [RS:1;80666a11a09f:39323 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-05T12:01:15,835 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T12:01:15,837 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-05T12:01:15,841 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T12:01:15,841 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:01:15,842 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T12:01:15,842 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-05T12:01:15,845 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-05T12:01:15,845 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:01:15,846 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T12:01:15,846 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-05T12:01:15,850 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T12:01:15,850 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:01:15,851 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T12:01:15,851 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-05T12:01:15,854 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T12:01:15,854 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:01:15,854 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T12:01:15,854 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-05T12:01:15,855 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740 2024-12-05T12:01:15,855 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740 2024-12-05T12:01:15,856 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-05T12:01:15,856 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-05T12:01:15,856 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T12:01:15,857 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-05T12:01:15,867 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T12:01:15,867 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72624343, jitterRate=0.08218704164028168}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T12:01:15,868 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733400075835Initializing all the Stores at 1733400075836 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733400075836Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733400075837 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733400075837Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733400075837Cleaning up temporary data from old regions at 1733400075856 (+19 ms)Region opened successfully at 1733400075868 (+12 ms) 2024-12-05T12:01:15,868 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-05T12:01:15,869 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-05T12:01:15,869 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-05T12:01:15,869 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T12:01:15,869 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T12:01:15,871 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-05T12:01:15,871 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733400075868Disabling compacts and flushes for region at 1733400075868Disabling writes for close at 1733400075869 (+1 ms)Writing region close event to WAL at 1733400075871 (+2 ms)Closed at 1733400075871 2024-12-05T12:01:15,873 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T12:01:15,873 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-05T12:01:15,873 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-05T12:01:15,874 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T12:01:15,876 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-05T12:01:15,915 DEBUG [RS:0;80666a11a09f:40679 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-05T12:01:15,915 DEBUG [RS:0;80666a11a09f:40679 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-05T12:01:15,915 DEBUG [RS:1;80666a11a09f:39323 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-05T12:01:15,915 DEBUG [RS:1;80666a11a09f:39323 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-05T12:01:15,976 DEBUG [RS:0;80666a11a09f:40679 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-05T12:01:15,977 DEBUG [RS:0;80666a11a09f:40679 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48787ed1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=80666a11a09f/172.17.0.2:0 2024-12-05T12:01:15,977 DEBUG [RS:1;80666a11a09f:39323 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-05T12:01:15,978 DEBUG [RS:1;80666a11a09f:39323 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fee78e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=80666a11a09f/172.17.0.2:0 2024-12-05T12:01:15,993 DEBUG [RS:0;80666a11a09f:40679 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;80666a11a09f:40679 2024-12-05T12:01:15,993 INFO [RS:0;80666a11a09f:40679 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-05T12:01:15,993 INFO [RS:0;80666a11a09f:40679 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-05T12:01:15,993 DEBUG [RS:0;80666a11a09f:40679 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-05T12:01:15,994 DEBUG [RS:1;80666a11a09f:39323 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;80666a11a09f:39323 2024-12-05T12:01:15,994 INFO [RS:1;80666a11a09f:39323 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-05T12:01:15,994 INFO [RS:1;80666a11a09f:39323 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-05T12:01:15,994 DEBUG [RS:1;80666a11a09f:39323 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-05T12:01:15,994 INFO [RS:0;80666a11a09f:40679 {}] regionserver.HRegionServer(2659): reportForDuty to master=80666a11a09f,46679,1733400075110 with port=40679, startcode=1733400075251 2024-12-05T12:01:15,995 DEBUG [RS:0;80666a11a09f:40679 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-05T12:01:15,999 INFO [RS:1;80666a11a09f:39323 {}] regionserver.HRegionServer(2659): reportForDuty to master=80666a11a09f,46679,1733400075110 with port=39323, startcode=1733400075331 2024-12-05T12:01:15,999 DEBUG [RS:1;80666a11a09f:39323 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-05T12:01:16,001 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38375, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-05T12:01:16,002 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46679 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 80666a11a09f,39323,1733400075331 2024-12-05T12:01:16,002 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46679 {}] master.ServerManager(517): Registering regionserver=80666a11a09f,39323,1733400075331 2024-12-05T12:01:16,003 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44149, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-05T12:01:16,004 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46679 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 80666a11a09f,40679,1733400075251 2024-12-05T12:01:16,004 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46679 {}] master.ServerManager(517): Registering regionserver=80666a11a09f,40679,1733400075251 2024-12-05T12:01:16,004 DEBUG [RS:1;80666a11a09f:39323 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6 2024-12-05T12:01:16,004 DEBUG [RS:1;80666a11a09f:39323 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-05T12:01:16,004 DEBUG [RS:1;80666a11a09f:39323 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-05T12:01:16,006 DEBUG [RS:0;80666a11a09f:40679 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6 2024-12-05T12:01:16,006 DEBUG [RS:0;80666a11a09f:40679 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-05T12:01:16,006 DEBUG [RS:0;80666a11a09f:40679 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-05T12:01:16,014 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46679-0x101a6a7bdc80000, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T12:01:16,026 WARN [80666a11a09f:46679 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-05T12:01:16,050 DEBUG [RS:1;80666a11a09f:39323 {}] zookeeper.ZKUtil(111): regionserver:39323-0x101a6a7bdc80002, quorum=127.0.0.1:63500, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/80666a11a09f,39323,1733400075331 2024-12-05T12:01:16,050 WARN [RS:1;80666a11a09f:39323 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T12:01:16,050 INFO [RS:1;80666a11a09f:39323 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T12:01:16,051 DEBUG [RS:1;80666a11a09f:39323 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/WALs/80666a11a09f,39323,1733400075331 2024-12-05T12:01:16,052 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [80666a11a09f,39323,1733400075331] 2024-12-05T12:01:16,052 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [80666a11a09f,40679,1733400075251] 2024-12-05T12:01:16,054 DEBUG [RS:0;80666a11a09f:40679 {}] zookeeper.ZKUtil(111): regionserver:40679-0x101a6a7bdc80001, quorum=127.0.0.1:63500, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/80666a11a09f,40679,1733400075251 2024-12-05T12:01:16,054 WARN [RS:0;80666a11a09f:40679 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T12:01:16,055 INFO [RS:0;80666a11a09f:40679 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T12:01:16,055 DEBUG [RS:0;80666a11a09f:40679 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/WALs/80666a11a09f,40679,1733400075251 2024-12-05T12:01:16,056 INFO [RS:1;80666a11a09f:39323 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-05T12:01:16,059 INFO [RS:1;80666a11a09f:39323 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-05T12:01:16,060 INFO [RS:1;80666a11a09f:39323 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-05T12:01:16,060 INFO [RS:1;80666a11a09f:39323 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:16,061 INFO [RS:1;80666a11a09f:39323 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-05T12:01:16,061 INFO [RS:1;80666a11a09f:39323 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-05T12:01:16,061 INFO [RS:1;80666a11a09f:39323 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:16,062 DEBUG [RS:1;80666a11a09f:39323 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:16,062 DEBUG [RS:1;80666a11a09f:39323 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:16,062 DEBUG [RS:1;80666a11a09f:39323 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:16,062 DEBUG [RS:1;80666a11a09f:39323 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:16,062 DEBUG [RS:1;80666a11a09f:39323 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:16,062 DEBUG [RS:1;80666a11a09f:39323 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/80666a11a09f:0, corePoolSize=2, maxPoolSize=2 2024-12-05T12:01:16,062 DEBUG [RS:1;80666a11a09f:39323 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:16,062 DEBUG [RS:1;80666a11a09f:39323 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:16,062 DEBUG [RS:1;80666a11a09f:39323 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:16,062 INFO [RS:0;80666a11a09f:40679 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-05T12:01:16,063 DEBUG [RS:1;80666a11a09f:39323 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:16,063 DEBUG [RS:1;80666a11a09f:39323 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:16,063 DEBUG [RS:1;80666a11a09f:39323 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:16,063 DEBUG [RS:1;80666a11a09f:39323 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/80666a11a09f:0, corePoolSize=3, maxPoolSize=3 2024-12-05T12:01:16,063 DEBUG [RS:1;80666a11a09f:39323 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0, corePoolSize=3, maxPoolSize=3 2024-12-05T12:01:16,065 INFO [RS:1;80666a11a09f:39323 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:16,065 INFO [RS:1;80666a11a09f:39323 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:16,065 INFO [RS:1;80666a11a09f:39323 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:16,065 INFO [RS:1;80666a11a09f:39323 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:16,065 INFO [RS:1;80666a11a09f:39323 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:16,065 INFO [RS:1;80666a11a09f:39323 {}] hbase.ChoreService(168): Chore ScheduledChore name=80666a11a09f,39323,1733400075331-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T12:01:16,065 INFO [RS:0;80666a11a09f:40679 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-05T12:01:16,066 INFO [RS:0;80666a11a09f:40679 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-05T12:01:16,066 INFO [RS:0;80666a11a09f:40679 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:16,066 INFO [RS:0;80666a11a09f:40679 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-05T12:01:16,067 INFO [RS:0;80666a11a09f:40679 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-05T12:01:16,067 INFO [RS:0;80666a11a09f:40679 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:16,067 DEBUG [RS:0;80666a11a09f:40679 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:16,067 DEBUG [RS:0;80666a11a09f:40679 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:16,067 DEBUG [RS:0;80666a11a09f:40679 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:16,067 DEBUG [RS:0;80666a11a09f:40679 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:16,067 DEBUG [RS:0;80666a11a09f:40679 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:16,067 DEBUG [RS:0;80666a11a09f:40679 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/80666a11a09f:0, corePoolSize=2, maxPoolSize=2 2024-12-05T12:01:16,068 DEBUG [RS:0;80666a11a09f:40679 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:16,068 DEBUG [RS:0;80666a11a09f:40679 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:16,068 DEBUG [RS:0;80666a11a09f:40679 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:16,068 DEBUG [RS:0;80666a11a09f:40679 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:16,068 DEBUG [RS:0;80666a11a09f:40679 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:16,068 DEBUG [RS:0;80666a11a09f:40679 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:16,068 DEBUG [RS:0;80666a11a09f:40679 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/80666a11a09f:0, corePoolSize=3, maxPoolSize=3 2024-12-05T12:01:16,068 DEBUG [RS:0;80666a11a09f:40679 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0, corePoolSize=3, maxPoolSize=3 2024-12-05T12:01:16,078 INFO [RS:0;80666a11a09f:40679 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:16,079 INFO [RS:0;80666a11a09f:40679 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:16,079 INFO [RS:0;80666a11a09f:40679 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:16,079 INFO [RS:0;80666a11a09f:40679 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:16,079 INFO [RS:0;80666a11a09f:40679 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:16,079 INFO [RS:0;80666a11a09f:40679 {}] hbase.ChoreService(168): Chore ScheduledChore name=80666a11a09f,40679,1733400075251-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T12:01:16,086 INFO [RS:1;80666a11a09f:39323 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-05T12:01:16,086 INFO [RS:1;80666a11a09f:39323 {}] hbase.ChoreService(168): Chore ScheduledChore name=80666a11a09f,39323,1733400075331-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:16,086 INFO [RS:1;80666a11a09f:39323 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:16,086 INFO [RS:1;80666a11a09f:39323 {}] regionserver.Replication(171): 80666a11a09f,39323,1733400075331 started 2024-12-05T12:01:16,100 INFO [RS:0;80666a11a09f:40679 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-05T12:01:16,100 INFO [RS:0;80666a11a09f:40679 {}] hbase.ChoreService(168): Chore ScheduledChore name=80666a11a09f,40679,1733400075251-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:16,100 INFO [RS:0;80666a11a09f:40679 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:16,100 INFO [RS:0;80666a11a09f:40679 {}] regionserver.Replication(171): 80666a11a09f,40679,1733400075251 started 2024-12-05T12:01:16,106 INFO [RS:1;80666a11a09f:39323 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:16,106 INFO [RS:1;80666a11a09f:39323 {}] regionserver.HRegionServer(1482): Serving as 80666a11a09f,39323,1733400075331, RpcServer on 80666a11a09f/172.17.0.2:39323, sessionid=0x101a6a7bdc80002 2024-12-05T12:01:16,106 DEBUG [RS:1;80666a11a09f:39323 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-05T12:01:16,106 DEBUG [RS:1;80666a11a09f:39323 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 80666a11a09f,39323,1733400075331 2024-12-05T12:01:16,106 DEBUG [RS:1;80666a11a09f:39323 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '80666a11a09f,39323,1733400075331' 2024-12-05T12:01:16,106 DEBUG [RS:1;80666a11a09f:39323 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-05T12:01:16,107 DEBUG [RS:1;80666a11a09f:39323 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-05T12:01:16,108 DEBUG [RS:1;80666a11a09f:39323 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-05T12:01:16,108 DEBUG [RS:1;80666a11a09f:39323 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-05T12:01:16,108 DEBUG [RS:1;80666a11a09f:39323 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 80666a11a09f,39323,1733400075331 2024-12-05T12:01:16,108 DEBUG [RS:1;80666a11a09f:39323 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '80666a11a09f,39323,1733400075331' 2024-12-05T12:01:16,108 DEBUG [RS:1;80666a11a09f:39323 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-05T12:01:16,108 DEBUG [RS:1;80666a11a09f:39323 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-05T12:01:16,109 DEBUG [RS:1;80666a11a09f:39323 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-05T12:01:16,109 INFO [RS:1;80666a11a09f:39323 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-05T12:01:16,109 INFO [RS:1;80666a11a09f:39323 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-05T12:01:16,116 INFO [RS:0;80666a11a09f:40679 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:16,116 INFO [RS:0;80666a11a09f:40679 {}] regionserver.HRegionServer(1482): Serving as 80666a11a09f,40679,1733400075251, RpcServer on 80666a11a09f/172.17.0.2:40679, sessionid=0x101a6a7bdc80001 2024-12-05T12:01:16,117 DEBUG [RS:0;80666a11a09f:40679 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-05T12:01:16,117 DEBUG [RS:0;80666a11a09f:40679 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 80666a11a09f,40679,1733400075251 2024-12-05T12:01:16,117 DEBUG [RS:0;80666a11a09f:40679 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '80666a11a09f,40679,1733400075251' 2024-12-05T12:01:16,117 DEBUG [RS:0;80666a11a09f:40679 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-05T12:01:16,117 DEBUG [RS:0;80666a11a09f:40679 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-05T12:01:16,118 DEBUG [RS:0;80666a11a09f:40679 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-05T12:01:16,118 DEBUG [RS:0;80666a11a09f:40679 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-05T12:01:16,118 DEBUG [RS:0;80666a11a09f:40679 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 80666a11a09f,40679,1733400075251 2024-12-05T12:01:16,118 DEBUG [RS:0;80666a11a09f:40679 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '80666a11a09f,40679,1733400075251' 2024-12-05T12:01:16,118 DEBUG [RS:0;80666a11a09f:40679 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-05T12:01:16,118 DEBUG [RS:0;80666a11a09f:40679 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-05T12:01:16,119 DEBUG [RS:0;80666a11a09f:40679 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-05T12:01:16,119 INFO [RS:0;80666a11a09f:40679 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-05T12:01:16,119 INFO [RS:0;80666a11a09f:40679 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-05T12:01:16,210 INFO [RS:1;80666a11a09f:39323 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T12:01:16,211 INFO [RS:1;80666a11a09f:39323 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=80666a11a09f%2C39323%2C1733400075331, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/WALs/80666a11a09f,39323,1733400075331, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/oldWALs, maxLogs=55 2024-12-05T12:01:16,220 INFO [RS:0;80666a11a09f:40679 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T12:01:16,220 INFO [RS:0;80666a11a09f:40679 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=80666a11a09f%2C40679%2C1733400075251, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/WALs/80666a11a09f,40679,1733400075251, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/oldWALs, maxLogs=55 2024-12-05T12:01:16,232 INFO [RS:1;80666a11a09f:39323 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/WALs/80666a11a09f,39323,1733400075331/80666a11a09f%2C39323%2C1733400075331.1733400076212 2024-12-05T12:01:16,232 DEBUG [RS:1;80666a11a09f:39323 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-05T12:01:16,242 INFO [RS:0;80666a11a09f:40679 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/WALs/80666a11a09f,40679,1733400075251/80666a11a09f%2C40679%2C1733400075251.1733400076222 2024-12-05T12:01:16,243 DEBUG [RS:0;80666a11a09f:40679 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-05T12:01:16,276 DEBUG [80666a11a09f:46679 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-05T12:01:16,277 DEBUG [80666a11a09f:46679 {}] balancer.BalancerClusterState(204): Hosts are {80666a11a09f=0} racks are {/default-rack=0} 2024-12-05T12:01:16,277 DEBUG [80666a11a09f:46679 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-05T12:01:16,277 DEBUG [80666a11a09f:46679 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-05T12:01:16,277 DEBUG [80666a11a09f:46679 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-05T12:01:16,277 DEBUG [80666a11a09f:46679 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-05T12:01:16,277 INFO [80666a11a09f:46679 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-05T12:01:16,277 INFO [80666a11a09f:46679 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-05T12:01:16,277 DEBUG [80666a11a09f:46679 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-05T12:01:16,278 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=80666a11a09f,40679,1733400075251 2024-12-05T12:01:16,279 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 80666a11a09f,40679,1733400075251, state=OPENING 2024-12-05T12:01:16,293 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-05T12:01:16,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40679-0x101a6a7bdc80001, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:01:16,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46679-0x101a6a7bdc80000, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:01:16,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39323-0x101a6a7bdc80002, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:01:16,302 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T12:01:16,302 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T12:01:16,302 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T12:01:16,302 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=80666a11a09f,40679,1733400075251}] 2024-12-05T12:01:16,302 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T12:01:16,455 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-05T12:01:16,458 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46725, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-05T12:01:16,461 INFO [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-05T12:01:16,461 INFO [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T12:01:16,462 INFO [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-05T12:01:16,463 INFO [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=80666a11a09f%2C40679%2C1733400075251.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/WALs/80666a11a09f,40679,1733400075251, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/oldWALs, maxLogs=55 2024-12-05T12:01:16,486 INFO [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/WALs/80666a11a09f,40679,1733400075251/80666a11a09f%2C40679%2C1733400075251.meta.1733400076464.meta 2024-12-05T12:01:16,487 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-05T12:01:16,487 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-05T12:01:16,487 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-05T12:01:16,487 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-05T12:01:16,488 INFO [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-05T12:01:16,488 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-05T12:01:16,488 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T12:01:16,488 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-05T12:01:16,488 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-05T12:01:16,490 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-05T12:01:16,491 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T12:01:16,491 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:01:16,491 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T12:01:16,491 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-05T12:01:16,492 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-05T12:01:16,492 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:01:16,492 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T12:01:16,492 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-05T12:01:16,492 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T12:01:16,492 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:01:16,493 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T12:01:16,493 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-05T12:01:16,493 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T12:01:16,493 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:01:16,493 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T12:01:16,494 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-05T12:01:16,495 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740 2024-12-05T12:01:16,495 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740 2024-12-05T12:01:16,495 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-05T12:01:16,495 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-05T12:01:16,496 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T12:01:16,497 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-05T12:01:16,497 INFO [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69802314, jitterRate=0.04013553261756897}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T12:01:16,497 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-05T12:01:16,498 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733400076488Writing region info on filesystem at 1733400076488Initializing all the Stores at 1733400076489 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733400076489Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733400076490 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733400076490Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733400076490Cleaning up temporary data from old regions at 1733400076496 (+6 ms)Running coprocessor post-open hooks at 1733400076498 (+2 ms)Region opened successfully at 1733400076498 2024-12-05T12:01:16,499 INFO [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733400076455 2024-12-05T12:01:16,502 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-05T12:01:16,502 INFO [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-05T12:01:16,502 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=80666a11a09f,40679,1733400075251 2024-12-05T12:01:16,503 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 80666a11a09f,40679,1733400075251, state=OPEN 2024-12-05T12:01:16,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40679-0x101a6a7bdc80001, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T12:01:16,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46679-0x101a6a7bdc80000, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T12:01:16,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39323-0x101a6a7bdc80002, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T12:01:16,509 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T12:01:16,509 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T12:01:16,509 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=80666a11a09f,40679,1733400075251 2024-12-05T12:01:16,509 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T12:01:16,512 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-05T12:01:16,512 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=80666a11a09f,40679,1733400075251 in 207 msec 2024-12-05T12:01:16,514 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-05T12:01:16,514 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 639 msec 2024-12-05T12:01:16,515 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T12:01:16,515 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-05T12:01:16,518 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:01:16,518 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40679,1733400075251, seqNum=-1] 2024-12-05T12:01:16,518 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:01:16,520 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48537, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:01:16,527 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 763 msec 2024-12-05T12:01:16,527 INFO [master/80666a11a09f:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733400076527, completionTime=-1 2024-12-05T12:01:16,527 INFO [master/80666a11a09f:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-05T12:01:16,527 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T12:01:16,531 INFO [master/80666a11a09f:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-05T12:01:16,532 INFO [master/80666a11a09f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733400136532 2024-12-05T12:01:16,532 INFO [master/80666a11a09f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733400196532 2024-12-05T12:01:16,532 INFO [master/80666a11a09f:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 4 msec 2024-12-05T12:01:16,532 INFO [master/80666a11a09f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80666a11a09f,46679,1733400075110-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:16,532 INFO [master/80666a11a09f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80666a11a09f,46679,1733400075110-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:16,532 INFO [master/80666a11a09f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80666a11a09f,46679,1733400075110-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:16,532 INFO [master/80666a11a09f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-80666a11a09f:46679, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:16,532 INFO [master/80666a11a09f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:16,534 INFO [master/80666a11a09f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:16,536 DEBUG [master/80666a11a09f:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-05T12:01:16,542 INFO [master/80666a11a09f:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.092sec 2024-12-05T12:01:16,554 INFO [master/80666a11a09f:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-05T12:01:16,555 INFO [master/80666a11a09f:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-05T12:01:16,555 INFO [master/80666a11a09f:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-05T12:01:16,555 INFO [master/80666a11a09f:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-05T12:01:16,555 INFO [master/80666a11a09f:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-05T12:01:16,555 INFO [master/80666a11a09f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80666a11a09f,46679,1733400075110-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T12:01:16,555 INFO [master/80666a11a09f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80666a11a09f,46679,1733400075110-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-05T12:01:16,556 DEBUG [master/80666a11a09f:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-05T12:01:16,556 INFO [master/80666a11a09f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-05T12:01:16,556 INFO [master/80666a11a09f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80666a11a09f,46679,1733400075110-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:16,604 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b10ca26, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:01:16,604 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80666a11a09f,46679,-1 for getting cluster id 2024-12-05T12:01:16,604 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T12:01:16,606 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e709e1b-2712-44bd-9a60-dab809f7a2fa' 2024-12-05T12:01:16,606 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T12:01:16,607 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e709e1b-2712-44bd-9a60-dab809f7a2fa" 2024-12-05T12:01:16,607 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f723663, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:01:16,607 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80666a11a09f,46679,-1] 2024-12-05T12:01:16,607 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T12:01:16,608 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:01:16,609 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34634, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T12:01:16,611 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d6c03d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T12:01:16,611 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:01:16,613 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40679,1733400075251, seqNum=-1] 2024-12-05T12:01:16,613 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:01:16,617 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47884, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:01:16,620 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=80666a11a09f,46679,1733400075110 2024-12-05T12:01:16,620 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T12:01:16,622 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncConnectionImpl(321): The fetched master address is 80666a11a09f,46679,1733400075110 2024-12-05T12:01:16,622 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5248647d 2024-12-05T12:01:16,622 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T12:01:16,625 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34650, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T12:01:16,626 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46679 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1733400076620', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T12:01:16,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46679 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733400076620 2024-12-05T12:01:16,628 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733400076620 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T12:01:16,628 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:01:16,629 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46679 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1733400076620" procId is: 4 2024-12-05T12:01:16,629 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733400076620 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T12:01:16,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46679 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T12:01:16,644 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733400076620-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => bdc2ef6e4cad9791ee7d6301ebcb5b3b, NAME => 'testBlockEvictionOnGracefulStop1733400076620,,1733400076625.bdc2ef6e4cad9791ee7d6301ebcb5b3b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1733400076620', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6 2024-12-05T12:01:16,654 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733400076620-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733400076620,,1733400076625.bdc2ef6e4cad9791ee7d6301ebcb5b3b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T12:01:16,654 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733400076620-pool-0 {}] regionserver.HRegion(1722): Closing bdc2ef6e4cad9791ee7d6301ebcb5b3b, disabling compactions & flushes 2024-12-05T12:01:16,654 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733400076620-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733400076620,,1733400076625.bdc2ef6e4cad9791ee7d6301ebcb5b3b. 2024-12-05T12:01:16,654 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733400076620-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733400076620,,1733400076625.bdc2ef6e4cad9791ee7d6301ebcb5b3b. 2024-12-05T12:01:16,654 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733400076620-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733400076620,,1733400076625.bdc2ef6e4cad9791ee7d6301ebcb5b3b. after waiting 0 ms 2024-12-05T12:01:16,654 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733400076620-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733400076620,,1733400076625.bdc2ef6e4cad9791ee7d6301ebcb5b3b. 2024-12-05T12:01:16,654 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733400076620-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733400076620,,1733400076625.bdc2ef6e4cad9791ee7d6301ebcb5b3b. 2024-12-05T12:01:16,654 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733400076620-pool-0 {}] regionserver.HRegion(1676): Region close journal for bdc2ef6e4cad9791ee7d6301ebcb5b3b: Waiting for close lock at 1733400076654Disabling compacts and flushes for region at 1733400076654Disabling writes for close at 1733400076654Writing region close event to WAL at 1733400076654Closed at 1733400076654 2024-12-05T12:01:16,656 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733400076620 execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T12:01:16,656 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1733400076620,,1733400076625.bdc2ef6e4cad9791ee7d6301ebcb5b3b.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1733400076656"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733400076656"}]},"ts":"1733400076656"} 2024-12-05T12:01:16,659 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T12:01:16,660 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733400076620 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T12:01:16,661 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1733400076620","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733400076661"}]},"ts":"1733400076661"} 2024-12-05T12:01:16,664 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1733400076620, state=ENABLING in hbase:meta 2024-12-05T12:01:16,664 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {80666a11a09f=0} racks are {/default-rack=0} 2024-12-05T12:01:16,666 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-05T12:01:16,666 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-05T12:01:16,666 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-05T12:01:16,666 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-05T12:01:16,666 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-05T12:01:16,666 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-05T12:01:16,666 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-05T12:01:16,666 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733400076620, region=bdc2ef6e4cad9791ee7d6301ebcb5b3b, ASSIGN}] 2024-12-05T12:01:16,670 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733400076620, region=bdc2ef6e4cad9791ee7d6301ebcb5b3b, ASSIGN 2024-12-05T12:01:16,672 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733400076620, region=bdc2ef6e4cad9791ee7d6301ebcb5b3b, ASSIGN; state=OFFLINE, location=80666a11a09f,40679,1733400075251; forceNewPlan=false, retain=false 2024-12-05T12:01:16,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46679 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T12:01:16,823 INFO [80666a11a09f:46679 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-05T12:01:16,823 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=bdc2ef6e4cad9791ee7d6301ebcb5b3b, regionState=OPENING, regionLocation=80666a11a09f,40679,1733400075251 2024-12-05T12:01:16,826 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733400076620, region=bdc2ef6e4cad9791ee7d6301ebcb5b3b, ASSIGN because future has completed 2024-12-05T12:01:16,827 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure bdc2ef6e4cad9791ee7d6301ebcb5b3b, server=80666a11a09f,40679,1733400075251}] 2024-12-05T12:01:16,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46679 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T12:01:16,981 INFO [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1733400076620,,1733400076625.bdc2ef6e4cad9791ee7d6301ebcb5b3b. 2024-12-05T12:01:16,982 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => bdc2ef6e4cad9791ee7d6301ebcb5b3b, NAME => 'testBlockEvictionOnGracefulStop1733400076620,,1733400076625.bdc2ef6e4cad9791ee7d6301ebcb5b3b.', STARTKEY => '', ENDKEY => ''} 2024-12-05T12:01:16,982 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1733400076620 bdc2ef6e4cad9791ee7d6301ebcb5b3b 2024-12-05T12:01:16,982 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733400076620,,1733400076625.bdc2ef6e4cad9791ee7d6301ebcb5b3b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T12:01:16,982 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for bdc2ef6e4cad9791ee7d6301ebcb5b3b 2024-12-05T12:01:16,982 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for bdc2ef6e4cad9791ee7d6301ebcb5b3b 2024-12-05T12:01:16,983 INFO [StoreOpener-bdc2ef6e4cad9791ee7d6301ebcb5b3b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region bdc2ef6e4cad9791ee7d6301ebcb5b3b 2024-12-05T12:01:16,987 INFO [StoreOpener-bdc2ef6e4cad9791ee7d6301ebcb5b3b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bdc2ef6e4cad9791ee7d6301ebcb5b3b columnFamilyName family 2024-12-05T12:01:16,987 DEBUG [StoreOpener-bdc2ef6e4cad9791ee7d6301ebcb5b3b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:01:16,987 INFO [StoreOpener-bdc2ef6e4cad9791ee7d6301ebcb5b3b-1 {}] regionserver.HStore(327): Store=bdc2ef6e4cad9791ee7d6301ebcb5b3b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T12:01:16,987 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for bdc2ef6e4cad9791ee7d6301ebcb5b3b 2024-12-05T12:01:16,988 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/default/testBlockEvictionOnGracefulStop1733400076620/bdc2ef6e4cad9791ee7d6301ebcb5b3b 2024-12-05T12:01:16,988 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/default/testBlockEvictionOnGracefulStop1733400076620/bdc2ef6e4cad9791ee7d6301ebcb5b3b 2024-12-05T12:01:16,988 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for bdc2ef6e4cad9791ee7d6301ebcb5b3b 2024-12-05T12:01:16,988 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for bdc2ef6e4cad9791ee7d6301ebcb5b3b 2024-12-05T12:01:16,989 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for bdc2ef6e4cad9791ee7d6301ebcb5b3b 2024-12-05T12:01:16,996 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/default/testBlockEvictionOnGracefulStop1733400076620/bdc2ef6e4cad9791ee7d6301ebcb5b3b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T12:01:16,997 INFO [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened bdc2ef6e4cad9791ee7d6301ebcb5b3b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67696218, jitterRate=0.008752256631851196}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T12:01:16,997 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bdc2ef6e4cad9791ee7d6301ebcb5b3b 2024-12-05T12:01:16,998 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for bdc2ef6e4cad9791ee7d6301ebcb5b3b: Running coprocessor pre-open hook at 1733400076983Writing region info on filesystem at 1733400076983Initializing all the Stores at 1733400076983Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733400076983Cleaning up temporary data from old regions at 1733400076988 (+5 ms)Running coprocessor post-open hooks at 1733400076997 (+9 ms)Region opened successfully at 1733400076997 2024-12-05T12:01:16,998 INFO [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1733400076620,,1733400076625.bdc2ef6e4cad9791ee7d6301ebcb5b3b., pid=6, masterSystemTime=1733400076979 2024-12-05T12:01:17,002 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1733400076620,,1733400076625.bdc2ef6e4cad9791ee7d6301ebcb5b3b. 2024-12-05T12:01:17,002 INFO [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1733400076620,,1733400076625.bdc2ef6e4cad9791ee7d6301ebcb5b3b. 2024-12-05T12:01:17,002 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=bdc2ef6e4cad9791ee7d6301ebcb5b3b, regionState=OPEN, openSeqNum=2, regionLocation=80666a11a09f,40679,1733400075251 2024-12-05T12:01:17,006 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure bdc2ef6e4cad9791ee7d6301ebcb5b3b, server=80666a11a09f,40679,1733400075251 because future has completed 2024-12-05T12:01:17,011 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-05T12:01:17,011 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure bdc2ef6e4cad9791ee7d6301ebcb5b3b, server=80666a11a09f,40679,1733400075251 in 182 msec 2024-12-05T12:01:17,013 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-05T12:01:17,013 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733400076620, region=bdc2ef6e4cad9791ee7d6301ebcb5b3b, ASSIGN in 345 msec 2024-12-05T12:01:17,014 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733400076620 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T12:01:17,014 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1733400076620","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733400077014"}]},"ts":"1733400077014"} 2024-12-05T12:01:17,017 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1733400076620, state=ENABLED in hbase:meta 2024-12-05T12:01:17,019 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733400076620 execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T12:01:17,021 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733400076620 in 393 msec 2024-12-05T12:01:17,181 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T12:01:17,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46679 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T12:01:17,256 INFO [RPCClient-NioEventLoopGroup-5-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1733400076620 completed 2024-12-05T12:01:17,256 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1733400076620 get assigned. Timeout = 60000ms 2024-12-05T12:01:17,256 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T12:01:17,259 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1733400076620 assigned to meta. Checking AM states. 2024-12-05T12:01:17,260 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-05T12:01:17,260 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1733400076620 assigned. 2024-12-05T12:01:17,264 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1733400076620', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1733400076620,,1733400076625.bdc2ef6e4cad9791ee7d6301ebcb5b3b., hostname=80666a11a09f,40679,1733400075251, seqNum=2] 2024-12-05T12:01:17,269 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing bdc2ef6e4cad9791ee7d6301ebcb5b3b 1/1 column families, dataSize=78 B heapSize=480 B 2024-12-05T12:01:17,277 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b6c4812cbea445f386d41e9eac2ff44a 2024-12-05T12:01:17,292 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/default/testBlockEvictionOnGracefulStop1733400076620/bdc2ef6e4cad9791ee7d6301ebcb5b3b/.tmp/family/b6c4812cbea445f386d41e9eac2ff44a is 43, key is row1/family:qf1/1/Put/seqid=0 2024-12-05T12:01:17,293 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/default/testBlockEvictionOnGracefulStop1733400076620/bdc2ef6e4cad9791ee7d6301ebcb5b3b/.tmp/family/b6c4812cbea445f386d41e9eac2ff44a 2024-12-05T12:01:17,297 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/default/testBlockEvictionOnGracefulStop1733400076620/bdc2ef6e4cad9791ee7d6301ebcb5b3b/.tmp/family/b6c4812cbea445f386d41e9eac2ff44a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/default/testBlockEvictionOnGracefulStop1733400076620/bdc2ef6e4cad9791ee7d6301ebcb5b3b/family/b6c4812cbea445f386d41e9eac2ff44a 2024-12-05T12:01:17,299 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/default/testBlockEvictionOnGracefulStop1733400076620/bdc2ef6e4cad9791ee7d6301ebcb5b3b/family/b6c4812cbea445f386d41e9eac2ff44a, delay=991 ms 2024-12-05T12:01:17,300 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/default/testBlockEvictionOnGracefulStop1733400076620/bdc2ef6e4cad9791ee7d6301ebcb5b3b/family/b6c4812cbea445f386d41e9eac2ff44a, entries=2, sequenceid=6, filesize=4.9 K 2024-12-05T12:01:17,301 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for bdc2ef6e4cad9791ee7d6301ebcb5b3b in 32ms, sequenceid=6, compaction requested=false 2024-12-05T12:01:17,301 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1733400076620' 2024-12-05T12:01:17,302 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for bdc2ef6e4cad9791ee7d6301ebcb5b3b: 2024-12-05T12:01:18,291 DEBUG [hfile-prefetch-1733400072870 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b6c4812cbea445f386d41e9eac2ff44a_0. 2024-12-05T12:01:18,291 DEBUG [hfile-prefetch-1733400072870 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b6c4812cbea445f386d41e9eac2ff44a_117. 2024-12-05T12:01:18,292 DEBUG [hfile-prefetch-1733400072870 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/default/testBlockEvictionOnGracefulStop1733400076620/bdc2ef6e4cad9791ee7d6301ebcb5b3b/family/b6c4812cbea445f386d41e9eac2ff44a, with total blocks 2, and data blocks 1 2024-12-05T12:01:18,292 DEBUG [hfile-prefetch-1733400072870 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-05T12:01:18,292 DEBUG [hfile-prefetch-1733400072870 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/default/testBlockEvictionOnGracefulStop1733400076620/bdc2ef6e4cad9791ee7d6301ebcb5b3b/family/b6c4812cbea445f386d41e9eac2ff44a, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-12-05T12:01:18,292 DEBUG [hfile-prefetch-1733400072870 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/default/testBlockEvictionOnGracefulStop1733400076620/bdc2ef6e4cad9791ee7d6301ebcb5b3b/family/b6c4812cbea445f386d41e9eac2ff44a. Will sleep 100ms and try the verification again. 2024-12-05T12:01:18,302 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;80666a11a09f:40679,5,FailOnTimeoutGroup] 2024-12-05T12:01:18,303 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '80666a11a09f,40679,1733400075251' ***** 2024-12-05T12:01:18,303 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-12-05T12:01:18,303 INFO [RS:0;80666a11a09f:40679 {}] bucket.BucketCache(1738): Disabling cache 2024-12-05T12:01:18,333 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-05T12:01:18,333 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-05T12:01:18,336 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1733400076620 2024-12-05T12:01:18,336 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1733400076620 Metrics about Tables on a single HBase RegionServer 2024-12-05T12:01:18,337 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-05T12:01:18,337 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-05T12:01:18,337 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-05T12:01:18,337 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-05T12:01:18,351 INFO [RS:0;80666a11a09f:40679 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4716cd85-2b6b-3a48-c1bf-78df3670c43e/bucket.persistence 2024-12-05T12:01:18,351 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-05T12:01:18,351 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-05T12:01:18,351 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-05T12:01:18,352 INFO [RS:0;80666a11a09f:40679 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-05T12:01:18,352 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-05T12:01:18,362 DEBUG [RS:0;80666a11a09f:40679 {}] bucket.BucketCache(1487): Thread RS:0;80666a11a09f:40679 started persisting bucket cache to file 2024-12-05T12:01:18,362 DEBUG [RS:0;80666a11a09f:40679 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-05T12:01:18,362 DEBUG [RS:0;80666a11a09f:40679 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-12-05T12:01:18,366 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-05T12:01:18,366 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-05T12:01:18,366 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-05T12:01:18,366 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-05T12:01:18,368 DEBUG [RS:0;80666a11a09f:40679 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-12-05T12:01:18,369 DEBUG [RS:0;80666a11a09f:40679 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1733400078362 2024-12-05T12:01:18,369 DEBUG [RS:0;80666a11a09f:40679 {}] bucket.BucketCache(1509): Thread RS:0;80666a11a09f:40679 finished persisting bucket cache to file, renaming 2024-12-05T12:01:18,369 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-05T12:01:18,369 INFO [RS:0;80666a11a09f:40679 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-05T12:01:18,369 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733400078366 2024-12-05T12:01:18,369 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-05T12:01:18,369 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-05T12:01:18,369 INFO [RS:0;80666a11a09f:40679 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-05T12:01:18,370 INFO [RS:0;80666a11a09f:40679 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-05T12:01:18,370 INFO [RS:0;80666a11a09f:40679 {}] regionserver.HRegionServer(3091): Received CLOSE for bdc2ef6e4cad9791ee7d6301ebcb5b3b 2024-12-05T12:01:18,370 INFO [RS:0;80666a11a09f:40679 {}] regionserver.HRegionServer(959): stopping server 80666a11a09f,40679,1733400075251 2024-12-05T12:01:18,370 INFO [RS:0;80666a11a09f:40679 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-05T12:01:18,371 INFO [RS:0;80666a11a09f:40679 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;80666a11a09f:40679. 2024-12-05T12:01:18,371 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1733400076620,,1733400076625.bdc2ef6e4cad9791ee7d6301ebcb5b3b. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-05T12:01:18,371 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing bdc2ef6e4cad9791ee7d6301ebcb5b3b, disabling compactions & flushes 2024-12-05T12:01:18,371 DEBUG [RS:0;80666a11a09f:40679 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T12:01:18,371 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733400076620,,1733400076625.bdc2ef6e4cad9791ee7d6301ebcb5b3b. 2024-12-05T12:01:18,371 DEBUG [RS:0;80666a11a09f:40679 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:01:18,371 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733400076620,,1733400076625.bdc2ef6e4cad9791ee7d6301ebcb5b3b. 2024-12-05T12:01:18,371 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733400076620,,1733400076625.bdc2ef6e4cad9791ee7d6301ebcb5b3b. after waiting 0 ms 2024-12-05T12:01:18,371 INFO [RS:0;80666a11a09f:40679 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-05T12:01:18,371 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733400076620,,1733400076625.bdc2ef6e4cad9791ee7d6301ebcb5b3b. 2024-12-05T12:01:18,371 INFO [RS:0;80666a11a09f:40679 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-05T12:01:18,371 INFO [RS:0;80666a11a09f:40679 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-05T12:01:18,371 INFO [RS:0;80666a11a09f:40679 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-05T12:01:18,371 INFO [RS:0;80666a11a09f:40679 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-05T12:01:18,371 DEBUG [RS:0;80666a11a09f:40679 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, bdc2ef6e4cad9791ee7d6301ebcb5b3b=testBlockEvictionOnGracefulStop1733400076620,,1733400076625.bdc2ef6e4cad9791ee7d6301ebcb5b3b.} 2024-12-05T12:01:18,371 INFO [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-05T12:01:18,371 DEBUG [RS:0;80666a11a09f:40679 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, bdc2ef6e4cad9791ee7d6301ebcb5b3b 2024-12-05T12:01:18,372 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-05T12:01:18,372 DEBUG [StoreFileCloser-bdc2ef6e4cad9791ee7d6301ebcb5b3b-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/default/testBlockEvictionOnGracefulStop1733400076620/bdc2ef6e4cad9791ee7d6301ebcb5b3b/family/b6c4812cbea445f386d41e9eac2ff44a 2024-12-05T12:01:18,372 INFO [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-05T12:01:18,372 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-05T12:01:18,372 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T12:01:18,372 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T12:01:18,372 WARN [hfile-prefetch-1733400072870 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/default/testBlockEvictionOnGracefulStop1733400076620/bdc2ef6e4cad9791ee7d6301ebcb5b3b/family/b6c4812cbea445f386d41e9eac2ff44a, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-12-05T12:01:18,372 INFO [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-12-05T12:01:18,372 DEBUG [hfile-prefetch-1733400072870 {}] hfile.PrefetchExecutor(121): Prefetch completed for b6c4812cbea445f386d41e9eac2ff44a 2024-12-05T12:01:18,376 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/default/testBlockEvictionOnGracefulStop1733400076620/bdc2ef6e4cad9791ee7d6301ebcb5b3b/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-05T12:01:18,377 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733400076620,,1733400076625.bdc2ef6e4cad9791ee7d6301ebcb5b3b. 2024-12-05T12:01:18,377 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for bdc2ef6e4cad9791ee7d6301ebcb5b3b: Waiting for close lock at 1733400078370Running coprocessor pre-close hooks at 1733400078371 (+1 ms)Disabling compacts and flushes for region at 1733400078371Disabling writes for close at 1733400078371Writing region close event to WAL at 1733400078372 (+1 ms)Running coprocessor post-close hooks at 1733400078377 (+5 ms)Closed at 1733400078377 2024-12-05T12:01:18,377 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1733400076620,,1733400076625.bdc2ef6e4cad9791ee7d6301ebcb5b3b. 2024-12-05T12:01:18,385 INFO [regionserver/80666a11a09f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T12:01:18,391 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 2ee64c37d82545b78d51276f112273f1 2024-12-05T12:01:18,403 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/.tmp/info/2ee64c37d82545b78d51276f112273f1 is 209, key is testBlockEvictionOnGracefulStop1733400076620,,1733400076625.bdc2ef6e4cad9791ee7d6301ebcb5b3b./info:regioninfo/1733400077002/Put/seqid=0 2024-12-05T12:01:18,404 INFO [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/.tmp/info/2ee64c37d82545b78d51276f112273f1 2024-12-05T12:01:18,416 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6327f0417fe4407eb77872bde4fba381 2024-12-05T12:01:18,433 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/.tmp/ns/6327f0417fe4407eb77872bde4fba381 is 43, key is default/ns:d/1733400076521/Put/seqid=0 2024-12-05T12:01:18,434 INFO [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/.tmp/ns/6327f0417fe4407eb77872bde4fba381 2024-12-05T12:01:18,443 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 0ab75bc082fd44a4bdcfe33c0d395e31 2024-12-05T12:01:18,461 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/.tmp/table/0ab75bc082fd44a4bdcfe33c0d395e31 is 80, key is testBlockEvictionOnGracefulStop1733400076620/table:state/1733400077014/Put/seqid=0 2024-12-05T12:01:18,461 INFO [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/.tmp/table/0ab75bc082fd44a4bdcfe33c0d395e31 2024-12-05T12:01:18,465 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/.tmp/info/2ee64c37d82545b78d51276f112273f1 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/info/2ee64c37d82545b78d51276f112273f1 2024-12-05T12:01:18,467 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/info/2ee64c37d82545b78d51276f112273f1, delay=939 ms 2024-12-05T12:01:18,469 INFO [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/info/2ee64c37d82545b78d51276f112273f1, entries=10, sequenceid=11, filesize=7.0 K 2024-12-05T12:01:18,469 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/.tmp/ns/6327f0417fe4407eb77872bde4fba381 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/ns/6327f0417fe4407eb77872bde4fba381 2024-12-05T12:01:18,471 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/ns/6327f0417fe4407eb77872bde4fba381, delay=983 ms 2024-12-05T12:01:18,472 INFO [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/ns/6327f0417fe4407eb77872bde4fba381, entries=2, sequenceid=11, filesize=5.0 K 2024-12-05T12:01:18,473 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/.tmp/table/0ab75bc082fd44a4bdcfe33c0d395e31 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/table/0ab75bc082fd44a4bdcfe33c0d395e31 2024-12-05T12:01:18,475 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/table/0ab75bc082fd44a4bdcfe33c0d395e31, delay=955 ms 2024-12-05T12:01:18,476 INFO [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/table/0ab75bc082fd44a4bdcfe33c0d395e31, entries=2, sequenceid=11, filesize=5.3 K 2024-12-05T12:01:18,477 INFO [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 105ms, sequenceid=11, compaction requested=false 2024-12-05T12:01:18,478 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/info/2ee64c37d82545b78d51276f112273f1 2024-12-05T12:01:18,478 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/ns/6327f0417fe4407eb77872bde4fba381 2024-12-05T12:01:18,479 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/table/0ab75bc082fd44a4bdcfe33c0d395e31 2024-12-05T12:01:18,485 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-12-05T12:01:18,485 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-05T12:01:18,485 INFO [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-05T12:01:18,486 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733400078371Running coprocessor pre-close hooks at 1733400078371Disabling compacts and flushes for region at 1733400078372 (+1 ms)Disabling writes for close at 1733400078372Obtaining lock to block concurrent updates at 1733400078372Preparing flush snapshotting stores in 1588230740 at 1733400078372Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1733400078372Flushing stores of hbase:meta,,1.1588230740 at 1733400078373 (+1 ms)Flushing 1588230740/info: creating writer at 1733400078373Flushing 1588230740/info: appending metadata at 1733400078403 (+30 ms)Flushing 1588230740/info: closing flushed file at 1733400078403Flushing 1588230740/ns: creating writer at 1733400078407 (+4 ms)Flushing 1588230740/ns: appending metadata at 1733400078433 (+26 ms)Flushing 1588230740/ns: closing flushed file at 1733400078433Flushing 1588230740/table: creating writer at 1733400078437 (+4 ms)Flushing 1588230740/table: appending metadata at 1733400078460 (+23 ms)Flushing 1588230740/table: closing flushed file at 1733400078460Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3dd97c86: reopening flushed file at 1733400078465 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c0794a0: reopening flushed file at 1733400078469 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2406406d: reopening flushed file at 1733400078473 (+4 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 105ms, sequenceid=11, compaction requested=false at 1733400078477 (+4 ms)Writing region close event to WAL at 1733400078479 (+2 ms)Running coprocessor post-close hooks at 1733400078485 (+6 ms)Closed at 1733400078485 2024-12-05T12:01:18,486 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-05T12:01:18,572 INFO [RS:0;80666a11a09f:40679 {}] regionserver.HRegionServer(976): stopping server 80666a11a09f,40679,1733400075251; all regions closed. 2024-12-05T12:01:18,574 DEBUG [RS:0;80666a11a09f:40679 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/oldWALs 2024-12-05T12:01:18,574 INFO [RS:0;80666a11a09f:40679 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 80666a11a09f%2C40679%2C1733400075251.meta:.meta(num 1733400076464) 2024-12-05T12:01:18,576 DEBUG [RS:0;80666a11a09f:40679 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/oldWALs 2024-12-05T12:01:18,576 INFO [RS:0;80666a11a09f:40679 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 80666a11a09f%2C40679%2C1733400075251:(num 1733400076222) 2024-12-05T12:01:18,576 DEBUG [RS:0;80666a11a09f:40679 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:01:18,576 INFO [RS:0;80666a11a09f:40679 {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T12:01:18,576 INFO [RS:0;80666a11a09f:40679 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-05T12:01:18,576 INFO [RS:0;80666a11a09f:40679 {}] hbase.ChoreService(370): Chore service for: regionserver/80666a11a09f:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-05T12:01:18,576 INFO [RS:0;80666a11a09f:40679 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-05T12:01:18,577 INFO [RS:0;80666a11a09f:40679 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40679 2024-12-05T12:01:18,578 INFO [regionserver/80666a11a09f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-05T12:01:18,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40679-0x101a6a7bdc80001, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/80666a11a09f,40679,1733400075251 2024-12-05T12:01:18,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46679-0x101a6a7bdc80000, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T12:01:18,593 INFO [RS:0;80666a11a09f:40679 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-05T12:01:18,594 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [80666a11a09f,40679,1733400075251] 2024-12-05T12:01:18,609 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/80666a11a09f,40679,1733400075251 already deleted, retry=false 2024-12-05T12:01:18,609 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 80666a11a09f,40679,1733400075251 on 80666a11a09f,46679,1733400075110 2024-12-05T12:01:18,614 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 80666a11a09f,40679,1733400075251, splitWal=true, meta=true 2024-12-05T12:01:18,617 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 80666a11a09f,40679,1733400075251 (carryingMeta=true) 80666a11a09f,40679,1733400075251/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@52e938d5[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-05T12:01:18,617 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 80666a11a09f,40679,1733400075251, splitWal=true, meta=true 2024-12-05T12:01:18,618 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 80666a11a09f,40679,1733400075251, splitWal=true, meta=true, isMeta: true 2024-12-05T12:01:18,619 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/WALs/80666a11a09f,40679,1733400075251-splitting 2024-12-05T12:01:18,619 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/WALs/80666a11a09f,40679,1733400075251-splitting dir is empty, no logs to split. 2024-12-05T12:01:18,619 INFO [PEWorker-1 {}] master.SplitWALManager(105): 80666a11a09f,40679,1733400075251 WAL count=0, meta=true 2024-12-05T12:01:18,622 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/WALs/80666a11a09f,40679,1733400075251-splitting dir is empty, no logs to split. 2024-12-05T12:01:18,622 INFO [PEWorker-1 {}] master.SplitWALManager(105): 80666a11a09f,40679,1733400075251 WAL count=0, meta=true 2024-12-05T12:01:18,622 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 80666a11a09f,40679,1733400075251 WAL splitting is done? wals=0, meta=true 2024-12-05T12:01:18,624 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-05T12:01:18,625 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T12:01:18,626 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-05T12:01:18,701 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40679-0x101a6a7bdc80001, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T12:01:18,701 INFO [RS:0;80666a11a09f:40679 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-05T12:01:18,701 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40679-0x101a6a7bdc80001, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T12:01:18,701 INFO [RS:0;80666a11a09f:40679 {}] regionserver.HRegionServer(1031): Exiting; stopping=80666a11a09f,40679,1733400075251; zookeeper connection closed. 2024-12-05T12:01:18,701 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@12cb7dc5 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@12cb7dc5 2024-12-05T12:01:18,776 DEBUG [80666a11a09f:46679 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-05T12:01:18,777 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=80666a11a09f,39323,1733400075331 2024-12-05T12:01:18,778 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 80666a11a09f,39323,1733400075331, state=OPENING 2024-12-05T12:01:18,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39323-0x101a6a7bdc80002, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T12:01:18,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46679-0x101a6a7bdc80000, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T12:01:18,784 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T12:01:18,784 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T12:01:18,784 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T12:01:18,784 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=80666a11a09f,39323,1733400075331}] 2024-12-05T12:01:18,819 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/80666a11a09f:0 server-side Connection retries=45 2024-12-05T12:01:18,819 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T12:01:18,819 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T12:01:18,819 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T12:01:18,819 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T12:01:18,819 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T12:01:18,819 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-05T12:01:18,819 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T12:01:18,820 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34025 2024-12-05T12:01:18,821 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34025 connecting to ZooKeeper ensemble=127.0.0.1:63500 2024-12-05T12:01:18,822 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T12:01:18,822 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T12:01:18,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:340250x0, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T12:01:18,835 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:34025-0x101a6a7bdc80003, quorum=127.0.0.1:63500, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-12-05T12:01:18,835 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34025-0x101a6a7bdc80003 connected 2024-12-05T12:01:18,835 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-12-05T12:01:18,836 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-05T12:01:18,837 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4716cd85-2b6b-3a48-c1bf-78df3670c43e/bucket.cache 2024-12-05T12:01:18,837 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-05T12:01:18,837 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-05T12:01:18,837 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-05T12:01:18,837 INFO [Thread-207 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-05T12:01:18,837 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4716cd85-2b6b-3a48-c1bf-78df3670c43e/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4716cd85-2b6b-3a48-c1bf-78df3670c43e/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-05T12:01:18,838 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-05T12:01:18,838 INFO [Thread-207 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-12-05T12:01:18,838 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:34025-0x101a6a7bdc80003, quorum=127.0.0.1:63500, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-05T12:01:18,839 INFO [Thread-207 {}] bucket.BucketCache(1705): Retrieved 0 of chunks with blockCount = 0. 2024-12-05T12:01:18,840 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34025-0x101a6a7bdc80003, quorum=127.0.0.1:63500, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T12:01:18,851 DEBUG [Thread-207 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-12-05T12:01:18,851 INFO [Thread-207 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-05T12:01:18,851 INFO [Thread-207 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 0 2024-12-05T12:01:18,851 INFO [Thread-207 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4716cd85-2b6b-3a48-c1bf-78df3670c43e/bucket.persistence is complete. 2024-12-05T12:01:18,853 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34025 2024-12-05T12:01:18,856 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34025 2024-12-05T12:01:18,858 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34025 2024-12-05T12:01:18,862 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34025 2024-12-05T12:01:18,864 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34025 2024-12-05T12:01:18,866 INFO [RS:2;80666a11a09f:34025 {}] regionserver.HRegionServer(746): ClusterId : 2e709e1b-2712-44bd-9a60-dab809f7a2fa 2024-12-05T12:01:18,867 DEBUG [RS:2;80666a11a09f:34025 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-05T12:01:18,877 DEBUG [RS:2;80666a11a09f:34025 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-05T12:01:18,877 DEBUG [RS:2;80666a11a09f:34025 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-05T12:01:18,885 DEBUG [RS:2;80666a11a09f:34025 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-05T12:01:18,886 DEBUG [RS:2;80666a11a09f:34025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7540d379, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=80666a11a09f/172.17.0.2:0 2024-12-05T12:01:18,904 DEBUG [RS:2;80666a11a09f:34025 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;80666a11a09f:34025 2024-12-05T12:01:18,904 INFO [RS:2;80666a11a09f:34025 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-05T12:01:18,904 INFO [RS:2;80666a11a09f:34025 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-05T12:01:18,904 DEBUG [RS:2;80666a11a09f:34025 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-05T12:01:18,905 INFO [RS:2;80666a11a09f:34025 {}] regionserver.HRegionServer(2659): reportForDuty to master=80666a11a09f,46679,1733400075110 with port=34025, startcode=1733400078818 2024-12-05T12:01:18,905 DEBUG [RS:2;80666a11a09f:34025 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-05T12:01:18,908 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45021, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-05T12:01:18,909 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46679 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 80666a11a09f,34025,1733400078818 2024-12-05T12:01:18,909 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46679 {}] master.ServerManager(517): Registering regionserver=80666a11a09f,34025,1733400078818 2024-12-05T12:01:18,910 DEBUG [RS:2;80666a11a09f:34025 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6 2024-12-05T12:01:18,911 DEBUG [RS:2;80666a11a09f:34025 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-05T12:01:18,911 DEBUG [RS:2;80666a11a09f:34025 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-05T12:01:18,918 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46679-0x101a6a7bdc80000, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T12:01:18,918 DEBUG [RS:2;80666a11a09f:34025 {}] zookeeper.ZKUtil(111): regionserver:34025-0x101a6a7bdc80003, quorum=127.0.0.1:63500, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/80666a11a09f,34025,1733400078818 2024-12-05T12:01:18,918 WARN [RS:2;80666a11a09f:34025 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T12:01:18,918 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [80666a11a09f,34025,1733400078818] 2024-12-05T12:01:18,918 INFO [RS:2;80666a11a09f:34025 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T12:01:18,919 DEBUG [RS:2;80666a11a09f:34025 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/WALs/80666a11a09f,34025,1733400078818 2024-12-05T12:01:18,924 INFO [RS:2;80666a11a09f:34025 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-05T12:01:18,926 INFO [RS:2;80666a11a09f:34025 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-05T12:01:18,931 INFO [RS:2;80666a11a09f:34025 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-05T12:01:18,931 INFO [RS:2;80666a11a09f:34025 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:18,931 INFO [RS:2;80666a11a09f:34025 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-05T12:01:18,932 INFO [RS:2;80666a11a09f:34025 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-05T12:01:18,932 INFO [RS:2;80666a11a09f:34025 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:18,932 DEBUG [RS:2;80666a11a09f:34025 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:18,932 DEBUG [RS:2;80666a11a09f:34025 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:18,932 DEBUG [RS:2;80666a11a09f:34025 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:18,932 DEBUG [RS:2;80666a11a09f:34025 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:18,932 DEBUG [RS:2;80666a11a09f:34025 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:18,932 DEBUG [RS:2;80666a11a09f:34025 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/80666a11a09f:0, corePoolSize=2, maxPoolSize=2 2024-12-05T12:01:18,932 DEBUG [RS:2;80666a11a09f:34025 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:18,932 DEBUG [RS:2;80666a11a09f:34025 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:18,932 DEBUG [RS:2;80666a11a09f:34025 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:18,932 DEBUG [RS:2;80666a11a09f:34025 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:18,933 DEBUG [RS:2;80666a11a09f:34025 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:18,933 DEBUG [RS:2;80666a11a09f:34025 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/80666a11a09f:0, corePoolSize=1, maxPoolSize=1 2024-12-05T12:01:18,933 DEBUG [RS:2;80666a11a09f:34025 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/80666a11a09f:0, corePoolSize=3, maxPoolSize=3 2024-12-05T12:01:18,933 DEBUG [RS:2;80666a11a09f:34025 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/80666a11a09f:0, corePoolSize=3, maxPoolSize=3 2024-12-05T12:01:18,938 INFO [RS:2;80666a11a09f:34025 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:18,938 INFO [RS:2;80666a11a09f:34025 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:18,939 INFO [RS:2;80666a11a09f:34025 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:18,939 INFO [RS:2;80666a11a09f:34025 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:18,939 INFO [RS:2;80666a11a09f:34025 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:18,939 INFO [RS:2;80666a11a09f:34025 {}] hbase.ChoreService(168): Chore ScheduledChore name=80666a11a09f,34025,1733400078818-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T12:01:18,940 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-05T12:01:18,942 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49097, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-05T12:01:18,947 INFO [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-05T12:01:18,947 INFO [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T12:01:18,947 INFO [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-05T12:01:18,949 INFO [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=80666a11a09f%2C39323%2C1733400075331.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/WALs/80666a11a09f,39323,1733400075331, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/oldWALs, maxLogs=55 2024-12-05T12:01:18,961 INFO [RS:2;80666a11a09f:34025 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-05T12:01:18,961 INFO [RS:2;80666a11a09f:34025 {}] hbase.ChoreService(168): Chore ScheduledChore name=80666a11a09f,34025,1733400078818-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:18,961 INFO [RS:2;80666a11a09f:34025 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:18,961 INFO [RS:2;80666a11a09f:34025 {}] regionserver.Replication(171): 80666a11a09f,34025,1733400078818 started 2024-12-05T12:01:18,973 INFO [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/WALs/80666a11a09f,39323,1733400075331/80666a11a09f%2C39323%2C1733400075331.meta.1733400078950.meta 2024-12-05T12:01:18,974 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-05T12:01:18,975 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-05T12:01:18,975 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-05T12:01:18,975 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-05T12:01:18,975 INFO [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-05T12:01:18,976 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-05T12:01:18,976 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T12:01:18,976 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-05T12:01:18,976 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-05T12:01:18,979 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-05T12:01:18,980 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T12:01:18,980 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:01:18,982 INFO [RS:2;80666a11a09f:34025 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T12:01:18,982 INFO [RS:2;80666a11a09f:34025 {}] regionserver.HRegionServer(1482): Serving as 80666a11a09f,34025,1733400078818, RpcServer on 80666a11a09f/172.17.0.2:34025, sessionid=0x101a6a7bdc80003 2024-12-05T12:01:18,983 DEBUG [RS:2;80666a11a09f:34025 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-05T12:01:18,983 DEBUG [RS:2;80666a11a09f:34025 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 80666a11a09f,34025,1733400078818 2024-12-05T12:01:18,983 DEBUG [RS:2;80666a11a09f:34025 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '80666a11a09f,34025,1733400078818' 2024-12-05T12:01:18,983 DEBUG [RS:2;80666a11a09f:34025 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-05T12:01:18,983 DEBUG [RS:2;80666a11a09f:34025 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-05T12:01:18,984 DEBUG [RS:2;80666a11a09f:34025 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-05T12:01:18,984 DEBUG [RS:2;80666a11a09f:34025 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-05T12:01:18,984 DEBUG [RS:2;80666a11a09f:34025 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 80666a11a09f,34025,1733400078818 2024-12-05T12:01:18,984 DEBUG [RS:2;80666a11a09f:34025 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '80666a11a09f,34025,1733400078818' 2024-12-05T12:01:18,984 DEBUG [RS:2;80666a11a09f:34025 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-05T12:01:18,985 DEBUG [RS:2;80666a11a09f:34025 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-05T12:01:18,986 DEBUG [RS:2;80666a11a09f:34025 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-05T12:01:18,986 INFO [RS:2;80666a11a09f:34025 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-05T12:01:18,986 INFO [RS:2;80666a11a09f:34025 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-05T12:01:18,991 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/info/2ee64c37d82545b78d51276f112273f1, delay=904 ms 2024-12-05T12:01:18,994 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/info/2ee64c37d82545b78d51276f112273f1 2024-12-05T12:01:18,994 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T12:01:18,995 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-05T12:01:18,996 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-05T12:01:18,996 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:01:18,998 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/ns/6327f0417fe4407eb77872bde4fba381, delay=989 ms 2024-12-05T12:01:18,999 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/ns/6327f0417fe4407eb77872bde4fba381 2024-12-05T12:01:19,000 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T12:01:19,000 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-05T12:01:19,001 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T12:01:19,001 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:01:19,001 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T12:01:19,001 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-05T12:01:19,002 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T12:01:19,002 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:01:19,005 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/table/0ab75bc082fd44a4bdcfe33c0d395e31, delay=958 ms 2024-12-05T12:01:19,009 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/table/0ab75bc082fd44a4bdcfe33c0d395e31 2024-12-05T12:01:19,009 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T12:01:19,010 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-05T12:01:19,010 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740 2024-12-05T12:01:19,011 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740 2024-12-05T12:01:19,011 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-05T12:01:19,011 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-05T12:01:19,012 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T12:01:19,013 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-05T12:01:19,013 INFO [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72135016, jitterRate=0.07489550113677979}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T12:01:19,014 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-05T12:01:19,014 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733400078976Writing region info on filesystem at 1733400078976Initializing all the Stores at 1733400078976Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733400078976Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733400078978 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733400078978Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733400078978Cleaning up temporary data from old regions at 1733400079011 (+33 ms)Running coprocessor post-open hooks at 1733400079014 (+3 ms)Region opened successfully at 1733400079014 2024-12-05T12:01:19,015 INFO [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1733400078939 2024-12-05T12:01:19,019 DEBUG [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-05T12:01:19,019 INFO [RS_OPEN_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-05T12:01:19,019 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=80666a11a09f,39323,1733400075331 2024-12-05T12:01:19,020 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 80666a11a09f,39323,1733400075331, state=OPEN 2024-12-05T12:01:19,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34025-0x101a6a7bdc80003, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T12:01:19,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39323-0x101a6a7bdc80002, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T12:01:19,026 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T12:01:19,026 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T12:01:19,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46679-0x101a6a7bdc80000, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T12:01:19,026 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T12:01:19,027 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=80666a11a09f,39323,1733400075331 2024-12-05T12:01:19,034 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-05T12:01:19,034 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 407 msec 2024-12-05T12:01:19,034 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-12-05T12:01:19,034 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=80666a11a09f,39323,1733400075331 in 243 msec 2024-12-05T12:01:19,034 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(207): 80666a11a09f,40679,1733400075251 had 2 regions 2024-12-05T12:01:19,035 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 80666a11a09f,40679,1733400075251, splitWal=true, meta=true, isMeta: false 2024-12-05T12:01:19,036 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/WALs/80666a11a09f,40679,1733400075251-splitting dir is empty, no logs to split. 2024-12-05T12:01:19,036 INFO [PEWorker-3 {}] master.SplitWALManager(105): 80666a11a09f,40679,1733400075251 WAL count=0, meta=false 2024-12-05T12:01:19,037 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/WALs/80666a11a09f,40679,1733400075251-splitting dir is empty, no logs to split. 2024-12-05T12:01:19,037 INFO [PEWorker-3 {}] master.SplitWALManager(105): 80666a11a09f,40679,1733400075251 WAL count=0, meta=false 2024-12-05T12:01:19,037 DEBUG [PEWorker-3 {}] procedure.ServerCrashProcedure(329): Check if 80666a11a09f,40679,1733400075251 WAL splitting is done? wals=0, meta=false 2024-12-05T12:01:19,039 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733400076620, region=bdc2ef6e4cad9791ee7d6301ebcb5b3b, ASSIGN}] 2024-12-05T12:01:19,041 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733400076620, region=bdc2ef6e4cad9791ee7d6301ebcb5b3b, ASSIGN 2024-12-05T12:01:19,042 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733400076620, region=bdc2ef6e4cad9791ee7d6301ebcb5b3b, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-05T12:01:19,087 INFO [RS:2;80666a11a09f:34025 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T12:01:19,088 INFO [RS:2;80666a11a09f:34025 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=80666a11a09f%2C34025%2C1733400078818, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/WALs/80666a11a09f,34025,1733400078818, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/oldWALs, maxLogs=55 2024-12-05T12:01:19,113 INFO [RS:2;80666a11a09f:34025 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/WALs/80666a11a09f,34025,1733400078818/80666a11a09f%2C34025%2C1733400078818.1733400079088 2024-12-05T12:01:19,113 DEBUG [RS:2;80666a11a09f:34025 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-05T12:01:19,193 DEBUG [80666a11a09f:46679 {}] balancer.BalancerClusterState(204): Hosts are {80666a11a09f=0} racks are {/default-rack=0} 2024-12-05T12:01:19,193 DEBUG [80666a11a09f:46679 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-05T12:01:19,193 DEBUG [80666a11a09f:46679 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-05T12:01:19,193 DEBUG [80666a11a09f:46679 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-05T12:01:19,193 DEBUG [80666a11a09f:46679 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-05T12:01:19,193 INFO [80666a11a09f:46679 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-05T12:01:19,193 INFO [80666a11a09f:46679 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-05T12:01:19,193 DEBUG [80666a11a09f:46679 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-05T12:01:19,194 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=bdc2ef6e4cad9791ee7d6301ebcb5b3b, regionState=OPENING, regionLocation=80666a11a09f,34025,1733400078818 2024-12-05T12:01:19,195 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 80666a11a09f:40679 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 80666a11a09f/172.17.0.2:40679 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T12:01:19,197 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40679,1733400075251, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40679,1733400075251, seqNum=-1, error=java.net.ConnectException: Call to address=80666a11a09f:40679 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 80666a11a09f/172.17.0.2:40679 2024-12-05T12:01:19,198 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40679,1733400075251, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-12-05T12:01:19,198 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=80666a11a09f,40679,1733400075251, seqNum=-1 from cache 2024-12-05T12:01:19,199 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.FailedServers(52): Added failed server with address 80666a11a09f:40679 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 80666a11a09f/172.17.0.2:40679 2024-12-05T12:01:19,304 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T12:01:19,304 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80666a11a09f,39323,1733400075331, seqNum=-1] 2024-12-05T12:01:19,305 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T12:01:19,306 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56987, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T12:01:19,308 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733400076620, region=bdc2ef6e4cad9791ee7d6301ebcb5b3b, ASSIGN because future has completed 2024-12-05T12:01:19,309 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure bdc2ef6e4cad9791ee7d6301ebcb5b3b, server=80666a11a09f,34025,1733400078818}] 2024-12-05T12:01:19,462 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-05T12:01:19,464 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-9-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45747, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-05T12:01:19,466 INFO [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1733400076620,,1733400076625.bdc2ef6e4cad9791ee7d6301ebcb5b3b. 2024-12-05T12:01:19,467 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7752): Opening region: {ENCODED => bdc2ef6e4cad9791ee7d6301ebcb5b3b, NAME => 'testBlockEvictionOnGracefulStop1733400076620,,1733400076625.bdc2ef6e4cad9791ee7d6301ebcb5b3b.', STARTKEY => '', ENDKEY => ''} 2024-12-05T12:01:19,468 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1733400076620 bdc2ef6e4cad9791ee7d6301ebcb5b3b 2024-12-05T12:01:19,468 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733400076620,,1733400076625.bdc2ef6e4cad9791ee7d6301ebcb5b3b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T12:01:19,468 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7794): checking encryption for bdc2ef6e4cad9791ee7d6301ebcb5b3b 2024-12-05T12:01:19,468 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7797): checking classloading for bdc2ef6e4cad9791ee7d6301ebcb5b3b 2024-12-05T12:01:19,469 INFO [StoreOpener-bdc2ef6e4cad9791ee7d6301ebcb5b3b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region bdc2ef6e4cad9791ee7d6301ebcb5b3b 2024-12-05T12:01:19,470 INFO [StoreOpener-bdc2ef6e4cad9791ee7d6301ebcb5b3b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bdc2ef6e4cad9791ee7d6301ebcb5b3b columnFamilyName family 2024-12-05T12:01:19,470 DEBUG [StoreOpener-bdc2ef6e4cad9791ee7d6301ebcb5b3b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T12:01:19,472 DEBUG [StoreFileOpener-bdc2ef6e4cad9791ee7d6301ebcb5b3b-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/default/testBlockEvictionOnGracefulStop1733400076620/bdc2ef6e4cad9791ee7d6301ebcb5b3b/family/b6c4812cbea445f386d41e9eac2ff44a, delay=964 ms 2024-12-05T12:01:19,474 DEBUG [StoreOpener-bdc2ef6e4cad9791ee7d6301ebcb5b3b-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/default/testBlockEvictionOnGracefulStop1733400076620/bdc2ef6e4cad9791ee7d6301ebcb5b3b/family/b6c4812cbea445f386d41e9eac2ff44a 2024-12-05T12:01:19,474 INFO [StoreOpener-bdc2ef6e4cad9791ee7d6301ebcb5b3b-1 {}] regionserver.HStore(327): Store=bdc2ef6e4cad9791ee7d6301ebcb5b3b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T12:01:19,474 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1038): replaying wal for bdc2ef6e4cad9791ee7d6301ebcb5b3b 2024-12-05T12:01:19,475 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/default/testBlockEvictionOnGracefulStop1733400076620/bdc2ef6e4cad9791ee7d6301ebcb5b3b 2024-12-05T12:01:19,475 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/default/testBlockEvictionOnGracefulStop1733400076620/bdc2ef6e4cad9791ee7d6301ebcb5b3b 2024-12-05T12:01:19,476 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1048): stopping wal replay for bdc2ef6e4cad9791ee7d6301ebcb5b3b 2024-12-05T12:01:19,476 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1060): Cleaning up temporary data for bdc2ef6e4cad9791ee7d6301ebcb5b3b 2024-12-05T12:01:19,477 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): writing seq id for bdc2ef6e4cad9791ee7d6301ebcb5b3b 2024-12-05T12:01:19,478 INFO [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1114): Opened bdc2ef6e4cad9791ee7d6301ebcb5b3b; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74038153, jitterRate=0.10325445234775543}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T12:01:19,478 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bdc2ef6e4cad9791ee7d6301ebcb5b3b 2024-12-05T12:01:19,480 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1006): Region open journal for bdc2ef6e4cad9791ee7d6301ebcb5b3b: Running coprocessor pre-open hook at 1733400079468Writing region info on filesystem at 1733400079468Initializing all the Stores at 1733400079468Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733400079469 (+1 ms)Cleaning up temporary data from old regions at 1733400079476 (+7 ms)Running coprocessor post-open hooks at 1733400079478 (+2 ms)Region opened successfully at 1733400079480 (+2 ms) 2024-12-05T12:01:19,482 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-05T12:01:19,482 INFO [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1733400076620,,1733400076625.bdc2ef6e4cad9791ee7d6301ebcb5b3b., pid=11, masterSystemTime=1733400079461 2024-12-05T12:01:19,483 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T12:01:19,483 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T12:01:19,483 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:01:19,483 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:01:19,483 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T12:01:19,483 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-05T12:01:19,483 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1433448022, stopped=false 2024-12-05T12:01:19,484 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=80666a11a09f,46679,1733400075110 2024-12-05T12:01:19,486 DEBUG [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1733400076620,,1733400076625.bdc2ef6e4cad9791ee7d6301ebcb5b3b. 2024-12-05T12:01:19,486 INFO [RS_OPEN_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1733400076620,,1733400076625.bdc2ef6e4cad9791ee7d6301ebcb5b3b. 2024-12-05T12:01:19,487 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=bdc2ef6e4cad9791ee7d6301ebcb5b3b, regionState=OPEN, openSeqNum=10, regionLocation=80666a11a09f,34025,1733400078818 2024-12-05T12:01:19,490 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure bdc2ef6e4cad9791ee7d6301ebcb5b3b, server=80666a11a09f,34025,1733400078818 because future has completed 2024-12-05T12:01:19,495 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-05T12:01:19,495 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure bdc2ef6e4cad9791ee7d6301ebcb5b3b, server=80666a11a09f,34025,1733400078818 in 183 msec 2024-12-05T12:01:19,497 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7 2024-12-05T12:01:19,497 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server 80666a11a09f,40679,1733400075251 after splitting done 2024-12-05T12:01:19,497 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733400076620, region=bdc2ef6e4cad9791ee7d6301ebcb5b3b, ASSIGN in 456 msec 2024-12-05T12:01:19,499 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 80666a11a09f,40679,1733400075251, splitWal=true, meta=true in 887 msec 2024-12-05T12:01:19,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34025-0x101a6a7bdc80003, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T12:01:19,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46679-0x101a6a7bdc80000, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T12:01:19,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46679-0x101a6a7bdc80000, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:01:19,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39323-0x101a6a7bdc80002, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T12:01:19,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39323-0x101a6a7bdc80002, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:01:19,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34025-0x101a6a7bdc80003, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:01:19,526 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T12:01:19,527 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T12:01:19,527 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T12:01:19,527 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39323-0x101a6a7bdc80002, quorum=127.0.0.1:63500, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T12:01:19,527 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34025-0x101a6a7bdc80003, quorum=127.0.0.1:63500, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T12:01:19,527 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:01:19,527 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '80666a11a09f,39323,1733400075331' ***** 2024-12-05T12:01:19,527 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-05T12:01:19,527 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '80666a11a09f,34025,1733400078818' ***** 2024-12-05T12:01:19,527 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-05T12:01:19,527 INFO [RS:2;80666a11a09f:34025 {}] bucket.BucketCache(1738): Disabling cache 2024-12-05T12:01:19,528 INFO [RS:2;80666a11a09f:34025 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4716cd85-2b6b-3a48-c1bf-78df3670c43e/bucket.persistence 2024-12-05T12:01:19,528 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-05T12:01:19,528 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-05T12:01:19,528 INFO [RS:1;80666a11a09f:39323 {}] bucket.BucketCache(1738): Disabling cache 2024-12-05T12:01:19,528 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-05T12:01:19,528 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-05T12:01:19,528 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-05T12:01:19,528 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-05T12:01:19,528 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46679-0x101a6a7bdc80000, quorum=127.0.0.1:63500, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T12:01:19,528 INFO [RS:2;80666a11a09f:34025 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-05T12:01:19,528 INFO [RS:1;80666a11a09f:39323 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4716cd85-2b6b-3a48-c1bf-78df3670c43e/bucket.persistence 2024-12-05T12:01:19,528 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-05T12:01:19,528 INFO [RS:1;80666a11a09f:39323 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-05T12:01:19,528 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-05T12:01:19,528 DEBUG [RS:2;80666a11a09f:34025 {}] bucket.BucketCache(1487): Thread RS:2;80666a11a09f:34025 started persisting bucket cache to file 2024-12-05T12:01:19,529 DEBUG [RS:2;80666a11a09f:34025 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-05T12:01:19,529 DEBUG [RS:2;80666a11a09f:34025 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-05T12:01:19,532 DEBUG [RS:2;80666a11a09f:34025 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-05T12:01:19,532 DEBUG [RS:2;80666a11a09f:34025 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733400079528 2024-12-05T12:01:19,532 DEBUG [RS:2;80666a11a09f:34025 {}] bucket.BucketCache(1509): Thread RS:2;80666a11a09f:34025 finished persisting bucket cache to file, renaming 2024-12-05T12:01:19,532 INFO [RS:2;80666a11a09f:34025 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-05T12:01:19,532 INFO [RS:2;80666a11a09f:34025 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-05T12:01:19,532 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-05T12:01:19,532 INFO [RS:2;80666a11a09f:34025 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-05T12:01:19,533 INFO [RS:2;80666a11a09f:34025 {}] regionserver.HRegionServer(3091): Received CLOSE for bdc2ef6e4cad9791ee7d6301ebcb5b3b 2024-12-05T12:01:19,533 INFO [RS:2;80666a11a09f:34025 {}] regionserver.HRegionServer(959): stopping server 80666a11a09f,34025,1733400078818 2024-12-05T12:01:19,533 INFO [RS:2;80666a11a09f:34025 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-05T12:01:19,533 INFO [RS:2;80666a11a09f:34025 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;80666a11a09f:34025. 2024-12-05T12:01:19,533 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1733400076620,,1733400076625.bdc2ef6e4cad9791ee7d6301ebcb5b3b. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-05T12:01:19,533 DEBUG [RS:2;80666a11a09f:34025 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T12:01:19,533 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing bdc2ef6e4cad9791ee7d6301ebcb5b3b, disabling compactions & flushes 2024-12-05T12:01:19,533 DEBUG [RS:2;80666a11a09f:34025 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:01:19,533 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733400076620,,1733400076625.bdc2ef6e4cad9791ee7d6301ebcb5b3b. 2024-12-05T12:01:19,533 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733400076620,,1733400076625.bdc2ef6e4cad9791ee7d6301ebcb5b3b. 2024-12-05T12:01:19,533 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733400076620,,1733400076625.bdc2ef6e4cad9791ee7d6301ebcb5b3b. after waiting 0 ms 2024-12-05T12:01:19,533 INFO [RS:2;80666a11a09f:34025 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-05T12:01:19,533 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733400076620,,1733400076625.bdc2ef6e4cad9791ee7d6301ebcb5b3b. 2024-12-05T12:01:19,533 DEBUG [RS:2;80666a11a09f:34025 {}] regionserver.HRegionServer(1325): Online Regions={bdc2ef6e4cad9791ee7d6301ebcb5b3b=testBlockEvictionOnGracefulStop1733400076620,,1733400076625.bdc2ef6e4cad9791ee7d6301ebcb5b3b.} 2024-12-05T12:01:19,533 DEBUG [RS:2;80666a11a09f:34025 {}] regionserver.HRegionServer(1351): Waiting on bdc2ef6e4cad9791ee7d6301ebcb5b3b 2024-12-05T12:01:19,534 DEBUG [StoreFileCloser-bdc2ef6e4cad9791ee7d6301ebcb5b3b-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/default/testBlockEvictionOnGracefulStop1733400076620/bdc2ef6e4cad9791ee7d6301ebcb5b3b/family/b6c4812cbea445f386d41e9eac2ff44a 2024-12-05T12:01:19,538 DEBUG [RS:1;80666a11a09f:39323 {}] bucket.BucketCache(1487): Thread RS:1;80666a11a09f:39323 started persisting bucket cache to file 2024-12-05T12:01:19,539 DEBUG [RS:1;80666a11a09f:39323 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-05T12:01:19,539 DEBUG [RS:1;80666a11a09f:39323 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-05T12:01:19,539 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/default/testBlockEvictionOnGracefulStop1733400076620/bdc2ef6e4cad9791ee7d6301ebcb5b3b/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-12-05T12:01:19,539 INFO [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733400076620,,1733400076625.bdc2ef6e4cad9791ee7d6301ebcb5b3b. 2024-12-05T12:01:19,539 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for bdc2ef6e4cad9791ee7d6301ebcb5b3b: Waiting for close lock at 1733400079533Running coprocessor pre-close hooks at 1733400079533Disabling compacts and flushes for region at 1733400079533Disabling writes for close at 1733400079533Writing region close event to WAL at 1733400079534 (+1 ms)Running coprocessor post-close hooks at 1733400079539 (+5 ms)Closed at 1733400079539 2024-12-05T12:01:19,539 DEBUG [RS_CLOSE_REGION-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1733400076620,,1733400076625.bdc2ef6e4cad9791ee7d6301ebcb5b3b. 2024-12-05T12:01:19,542 DEBUG [RS:1;80666a11a09f:39323 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-05T12:01:19,542 DEBUG [RS:1;80666a11a09f:39323 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733400079538 2024-12-05T12:01:19,542 DEBUG [RS:1;80666a11a09f:39323 {}] bucket.BucketCache(1509): Thread RS:1;80666a11a09f:39323 finished persisting bucket cache to file, renaming 2024-12-05T12:01:19,542 INFO [RS:1;80666a11a09f:39323 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-05T12:01:19,543 INFO [RS:1;80666a11a09f:39323 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-05T12:01:19,543 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-05T12:01:19,543 INFO [RS:1;80666a11a09f:39323 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-05T12:01:19,543 INFO [RS:1;80666a11a09f:39323 {}] regionserver.HRegionServer(959): stopping server 80666a11a09f,39323,1733400075331 2024-12-05T12:01:19,543 INFO [RS:1;80666a11a09f:39323 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-05T12:01:19,543 INFO [RS:1;80666a11a09f:39323 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;80666a11a09f:39323. 2024-12-05T12:01:19,543 DEBUG [RS:1;80666a11a09f:39323 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T12:01:19,543 DEBUG [RS:1;80666a11a09f:39323 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:01:19,543 INFO [RS:1;80666a11a09f:39323 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-05T12:01:19,543 INFO [RS:1;80666a11a09f:39323 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-05T12:01:19,543 INFO [RS:1;80666a11a09f:39323 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-05T12:01:19,543 INFO [RS:1;80666a11a09f:39323 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-05T12:01:19,543 INFO [RS:1;80666a11a09f:39323 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-05T12:01:19,543 DEBUG [RS:1;80666a11a09f:39323 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-05T12:01:19,544 DEBUG [RS:1;80666a11a09f:39323 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T12:01:19,544 INFO [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-05T12:01:19,544 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-05T12:01:19,544 INFO [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-05T12:01:19,544 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-05T12:01:19,544 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T12:01:19,544 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T12:01:19,544 INFO [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.21 KB heapSize=2.80 KB 2024-12-05T12:01:19,555 INFO [regionserver/80666a11a09f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T12:01:19,558 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on fb57a1c352e749afab9c48e68a601694 2024-12-05T12:01:19,568 INFO [regionserver/80666a11a09f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T12:01:19,570 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/.tmp/info/fb57a1c352e749afab9c48e68a601694 is 209, key is testBlockEvictionOnGracefulStop1733400076620,,1733400076625.bdc2ef6e4cad9791ee7d6301ebcb5b3b./info:regioninfo/1733400079486/Put/seqid=0 2024-12-05T12:01:19,571 INFO [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.21 KB at sequenceid=19 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/.tmp/info/fb57a1c352e749afab9c48e68a601694 2024-12-05T12:01:19,573 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/.tmp/info/fb57a1c352e749afab9c48e68a601694 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/info/fb57a1c352e749afab9c48e68a601694 2024-12-05T12:01:19,574 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/info/fb57a1c352e749afab9c48e68a601694, delay=999 ms 2024-12-05T12:01:19,575 INFO [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/info/fb57a1c352e749afab9c48e68a601694, entries=8, sequenceid=19, filesize=6.6 K 2024-12-05T12:01:19,576 INFO [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 32ms, sequenceid=19, compaction requested=false 2024-12-05T12:01:19,577 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/info/2ee64c37d82545b78d51276f112273f1 2024-12-05T12:01:19,578 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/info/fb57a1c352e749afab9c48e68a601694 2024-12-05T12:01:19,581 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/ns/6327f0417fe4407eb77872bde4fba381 2024-12-05T12:01:19,582 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/table/0ab75bc082fd44a4bdcfe33c0d395e31 2024-12-05T12:01:19,586 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/data/hbase/meta/1588230740/recovered.edits/22.seqid, newMaxSeqId=22, maxSeqId=14 2024-12-05T12:01:19,587 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-05T12:01:19,587 INFO [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-05T12:01:19,587 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733400079544Running coprocessor pre-close hooks at 1733400079544Disabling compacts and flushes for region at 1733400079544Disabling writes for close at 1733400079544Obtaining lock to block concurrent updates at 1733400079544Preparing flush snapshotting stores in 1588230740 at 1733400079544Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1238, getHeapSize=2800, getOffHeapSize=0, getCellsCount=8 at 1733400079545 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733400079545Flushing 1588230740/info: creating writer at 1733400079545Flushing 1588230740/info: appending metadata at 1733400079570 (+25 ms)Flushing 1588230740/info: closing flushed file at 1733400079570Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@142d71a3: reopening flushed file at 1733400079573 (+3 ms)Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 32ms, sequenceid=19, compaction requested=false at 1733400079576 (+3 ms)Writing region close event to WAL at 1733400079582 (+6 ms)Running coprocessor post-close hooks at 1733400079587 (+5 ms)Closed at 1733400079587 2024-12-05T12:01:19,587 DEBUG [RS_CLOSE_META-regionserver/80666a11a09f:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-05T12:01:19,734 INFO [RS:2;80666a11a09f:34025 {}] regionserver.HRegionServer(976): stopping server 80666a11a09f,34025,1733400078818; all regions closed. 2024-12-05T12:01:19,735 DEBUG [RS:2;80666a11a09f:34025 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/oldWALs 2024-12-05T12:01:19,735 INFO [RS:2;80666a11a09f:34025 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 80666a11a09f%2C34025%2C1733400078818:(num 1733400079088) 2024-12-05T12:01:19,736 DEBUG [RS:2;80666a11a09f:34025 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:01:19,736 INFO [RS:2;80666a11a09f:34025 {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T12:01:19,736 INFO [RS:2;80666a11a09f:34025 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-05T12:01:19,736 INFO [RS:2;80666a11a09f:34025 {}] hbase.ChoreService(370): Chore service for: regionserver/80666a11a09f:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-05T12:01:19,736 INFO [RS:2;80666a11a09f:34025 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-05T12:01:19,736 INFO [RS:2;80666a11a09f:34025 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-05T12:01:19,736 INFO [RS:2;80666a11a09f:34025 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-05T12:01:19,736 INFO [RS:2;80666a11a09f:34025 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-05T12:01:19,736 INFO [RS:2;80666a11a09f:34025 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34025 2024-12-05T12:01:19,737 INFO [regionserver/80666a11a09f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-05T12:01:19,744 INFO [RS:1;80666a11a09f:39323 {}] regionserver.HRegionServer(976): stopping server 80666a11a09f,39323,1733400075331; all regions closed. 2024-12-05T12:01:19,746 DEBUG [RS:1;80666a11a09f:39323 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/oldWALs 2024-12-05T12:01:19,746 INFO [RS:1;80666a11a09f:39323 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 80666a11a09f%2C39323%2C1733400075331.meta:.meta(num 1733400078950) 2024-12-05T12:01:19,747 DEBUG [RS:1;80666a11a09f:39323 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/oldWALs 2024-12-05T12:01:19,747 INFO [RS:1;80666a11a09f:39323 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 80666a11a09f%2C39323%2C1733400075331:(num 1733400076212) 2024-12-05T12:01:19,747 DEBUG [RS:1;80666a11a09f:39323 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T12:01:19,747 INFO [RS:1;80666a11a09f:39323 {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T12:01:19,747 INFO [RS:1;80666a11a09f:39323 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-05T12:01:19,748 INFO [RS:1;80666a11a09f:39323 {}] hbase.ChoreService(370): Chore service for: regionserver/80666a11a09f:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-05T12:01:19,748 INFO [RS:1;80666a11a09f:39323 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-05T12:01:19,748 INFO [regionserver/80666a11a09f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-05T12:01:19,748 INFO [RS:1;80666a11a09f:39323 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39323 2024-12-05T12:01:19,751 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34025-0x101a6a7bdc80003, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/80666a11a09f,34025,1733400078818 2024-12-05T12:01:19,751 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46679-0x101a6a7bdc80000, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T12:01:19,751 INFO [RS:2;80666a11a09f:34025 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-05T12:01:19,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39323-0x101a6a7bdc80002, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/80666a11a09f,39323,1733400075331 2024-12-05T12:01:19,759 INFO [RS:1;80666a11a09f:39323 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-05T12:01:19,760 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [80666a11a09f,39323,1733400075331] 2024-12-05T12:01:19,776 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/80666a11a09f,39323,1733400075331 already deleted, retry=false 2024-12-05T12:01:19,776 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 80666a11a09f,39323,1733400075331 expired; onlineServers=1 2024-12-05T12:01:19,776 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [80666a11a09f,34025,1733400078818] 2024-12-05T12:01:19,784 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/80666a11a09f,34025,1733400078818 already deleted, retry=false 2024-12-05T12:01:19,784 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 80666a11a09f,34025,1733400078818 expired; onlineServers=0 2024-12-05T12:01:19,784 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '80666a11a09f,46679,1733400075110' ***** 2024-12-05T12:01:19,784 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-05T12:01:19,784 INFO [M:0;80666a11a09f:46679 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-05T12:01:19,785 INFO [M:0;80666a11a09f:46679 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-05T12:01:19,785 DEBUG [M:0;80666a11a09f:46679 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-05T12:01:19,785 DEBUG [M:0;80666a11a09f:46679 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-05T12:01:19,785 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-05T12:01:19,785 DEBUG [master/80666a11a09f:0:becomeActiveMaster-HFileCleaner.large.0-1733400075787 {}] cleaner.HFileCleaner(306): Exit Thread[master/80666a11a09f:0:becomeActiveMaster-HFileCleaner.large.0-1733400075787,5,FailOnTimeoutGroup] 2024-12-05T12:01:19,785 DEBUG [master/80666a11a09f:0:becomeActiveMaster-HFileCleaner.small.0-1733400075791 {}] cleaner.HFileCleaner(306): Exit Thread[master/80666a11a09f:0:becomeActiveMaster-HFileCleaner.small.0-1733400075791,5,FailOnTimeoutGroup] 2024-12-05T12:01:19,785 INFO [M:0;80666a11a09f:46679 {}] hbase.ChoreService(370): Chore service for: master/80666a11a09f:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-05T12:01:19,785 INFO [M:0;80666a11a09f:46679 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-05T12:01:19,785 DEBUG [M:0;80666a11a09f:46679 {}] master.HMaster(1795): Stopping service threads 2024-12-05T12:01:19,785 INFO [M:0;80666a11a09f:46679 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-05T12:01:19,786 INFO [M:0;80666a11a09f:46679 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T12:01:19,786 INFO [M:0;80666a11a09f:46679 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T12:01:19,786 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-05T12:01:19,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46679-0x101a6a7bdc80000, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-05T12:01:19,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46679-0x101a6a7bdc80000, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T12:01:19,792 DEBUG [M:0;80666a11a09f:46679 {}] zookeeper.ZKUtil(347): master:46679-0x101a6a7bdc80000, quorum=127.0.0.1:63500, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-05T12:01:19,793 WARN [M:0;80666a11a09f:46679 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-05T12:01:19,793 INFO [M:0;80666a11a09f:46679 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/.lastflushedseqids 2024-12-05T12:01:19,796 INFO [M:0;80666a11a09f:46679 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T12:01:19,796 INFO [M:0;80666a11a09f:46679 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-05T12:01:19,796 DEBUG [M:0;80666a11a09f:46679 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-05T12:01:19,796 INFO [M:0;80666a11a09f:46679 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T12:01:19,796 DEBUG [M:0;80666a11a09f:46679 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T12:01:19,796 DEBUG [M:0;80666a11a09f:46679 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-05T12:01:19,796 DEBUG [M:0;80666a11a09f:46679 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T12:01:19,797 INFO [M:0;80666a11a09f:46679 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=39.50 KB heapSize=49.91 KB 2024-12-05T12:01:19,806 DEBUG [M:0;80666a11a09f:46679 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f50b105ba6dd44f8be54e4bc7bda1420 2024-12-05T12:01:19,819 DEBUG [M:0;80666a11a09f:46679 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f50b105ba6dd44f8be54e4bc7bda1420 is 82, key is hbase:meta,,1/info:regioninfo/1733400079019/Put/seqid=0 2024-12-05T12:01:19,820 INFO [M:0;80666a11a09f:46679 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f50b105ba6dd44f8be54e4bc7bda1420 2024-12-05T12:01:19,827 DEBUG [M:0;80666a11a09f:46679 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on dbd18a3a3e874a8f8712f0e994df8175 2024-12-05T12:01:19,840 DEBUG [M:0;80666a11a09f:46679 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/dbd18a3a3e874a8f8712f0e994df8175 is 782, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733400077021/Put/seqid=0 2024-12-05T12:01:19,840 INFO [M:0;80666a11a09f:46679 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.27 KB at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/dbd18a3a3e874a8f8712f0e994df8175 2024-12-05T12:01:19,847 DEBUG [M:0;80666a11a09f:46679 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 854749d2629e4519a7911b465702d653 2024-12-05T12:01:19,866 DEBUG [M:0;80666a11a09f:46679 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/854749d2629e4519a7911b465702d653 is 69, key is 80666a11a09f,34025,1733400078818/rs:state/1733400078909/Put/seqid=0 2024-12-05T12:01:19,867 INFO [M:0;80666a11a09f:46679 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/854749d2629e4519a7911b465702d653 2024-12-05T12:01:19,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34025-0x101a6a7bdc80003, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T12:01:19,868 INFO [RS:2;80666a11a09f:34025 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-05T12:01:19,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34025-0x101a6a7bdc80003, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T12:01:19,868 INFO [RS:2;80666a11a09f:34025 {}] regionserver.HRegionServer(1031): Exiting; stopping=80666a11a09f,34025,1733400078818; zookeeper connection closed. 2024-12-05T12:01:19,868 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5819bf99 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5819bf99 2024-12-05T12:01:19,869 INFO [M:0;80666a11a09f:46679 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 854749d2629e4519a7911b465702d653 2024-12-05T12:01:19,869 DEBUG [M:0;80666a11a09f:46679 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f50b105ba6dd44f8be54e4bc7bda1420 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f50b105ba6dd44f8be54e4bc7bda1420 2024-12-05T12:01:19,872 INFO [M:0;80666a11a09f:46679 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f50b105ba6dd44f8be54e4bc7bda1420, entries=14, sequenceid=102, filesize=5.9 K 2024-12-05T12:01:19,872 DEBUG [M:0;80666a11a09f:46679 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/dbd18a3a3e874a8f8712f0e994df8175 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/dbd18a3a3e874a8f8712f0e994df8175 2024-12-05T12:01:19,874 INFO [M:0;80666a11a09f:46679 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/dbd18a3a3e874a8f8712f0e994df8175, entries=11, sequenceid=102, filesize=6.5 K 2024-12-05T12:01:19,875 DEBUG [M:0;80666a11a09f:46679 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/854749d2629e4519a7911b465702d653 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/854749d2629e4519a7911b465702d653 2024-12-05T12:01:19,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39323-0x101a6a7bdc80002, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T12:01:19,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39323-0x101a6a7bdc80002, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T12:01:19,876 INFO [RS:1;80666a11a09f:39323 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-05T12:01:19,876 INFO [RS:1;80666a11a09f:39323 {}] regionserver.HRegionServer(1031): Exiting; stopping=80666a11a09f,39323,1733400075331; zookeeper connection closed. 2024-12-05T12:01:19,877 INFO [M:0;80666a11a09f:46679 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 854749d2629e4519a7911b465702d653 2024-12-05T12:01:19,877 INFO [M:0;80666a11a09f:46679 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7585d53c-2d13-2e8a-21c7-09a14760d506/84d9f050-7a3e-fc84-ee82-a3d9be9140e6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/854749d2629e4519a7911b465702d653, entries=3, sequenceid=102, filesize=5.3 K 2024-12-05T12:01:19,878 INFO [M:0;80666a11a09f:46679 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.50 KB/40448, heapSize ~49.61 KB/50800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 82ms, sequenceid=102, compaction requested=false 2024-12-05T12:01:19,878 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6bf5b05 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6bf5b05 2024-12-05T12:01:19,878 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-05T12:01:19,880 INFO [M:0;80666a11a09f:46679 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T12:01:19,880 DEBUG [M:0;80666a11a09f:46679 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733400079796Disabling compacts and flushes for region at 1733400079796Disabling writes for close at 1733400079796Obtaining lock to block concurrent updates at 1733400079797 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733400079797Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=40448, getHeapSize=51040, getOffHeapSize=0, getCellsCount=127 at 1733400079797Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733400079797Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733400079797Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733400079819 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733400079819Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733400079822 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733400079839 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733400079839Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733400079842 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733400079865 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733400079865Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ed61a2d: reopening flushed file at 1733400079869 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@84e842: reopening flushed file at 1733400079872 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37a3b89: reopening flushed file at 1733400079874 (+2 ms)Finished flush of dataSize ~39.50 KB/40448, heapSize ~49.61 KB/50800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 82ms, sequenceid=102, compaction requested=false at 1733400079878 (+4 ms)Writing region close event to WAL at 1733400079879 (+1 ms)Closed at 1733400079879 2024-12-05T12:01:19,886 INFO [M:0;80666a11a09f:46679 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-05T12:01:19,886 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-05T12:01:19,886 INFO [M:0;80666a11a09f:46679 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46679 2024-12-05T12:01:19,887 INFO [M:0;80666a11a09f:46679 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-05T12:01:19,993 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46679-0x101a6a7bdc80000, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T12:01:19,993 INFO [M:0;80666a11a09f:46679 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-05T12:01:19,993 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46679-0x101a6a7bdc80000, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T12:01:19,996 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-05T12:01:19,997 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-05T12:01:19,997 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T12:01:20,004 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=73 (was 51) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=999 (was 1016), ProcessCount=11 (was 11), AvailableMemoryMB=7306 (was 7381)