2024-12-04 15:44:45,160 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2
2024-12-04 15:44:45,179 main DEBUG Took 0.017489 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging
2024-12-04 15:44:45,179 main DEBUG PluginManager 'Core' found 129 plugins
2024-12-04 15:44:45,180 main DEBUG PluginManager 'Level' found 0 plugins
2024-12-04 15:44:45,180 main DEBUG PluginManager 'Lookup' found 16 plugins
2024-12-04 15:44:45,182 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-04 15:44:45,188 main DEBUG PluginManager 'TypeConverter' found 26 plugins
2024-12-04 15:44:45,200 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-04 15:44:45,202 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-04 15:44:45,202 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-04 15:44:45,202 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-04 15:44:45,203 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-04 15:44:45,203 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-04 15:44:45,204 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-04 15:44:45,204 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-04 15:44:45,205 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-04 15:44:45,205 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-04 15:44:45,206 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-04 15:44:45,206 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-04 15:44:45,207 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-04 15:44:45,207 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-04 15:44:45,207 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-04 15:44:45,208 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-04 15:44:45,208 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-04 15:44:45,209 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-04 15:44:45,209 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-04 15:44:45,209 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-04 15:44:45,210 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-04 15:44:45,210 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-04 15:44:45,210 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-04 15:44:45,211 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-12-04 15:44:45,211 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-04 15:44:45,211 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger].
2024-12-04 15:44:45,213 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-12-04 15:44:45,214 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin].
2024-12-04 15:44:45,215 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root})
2024-12-04 15:44:45,216 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout].
2024-12-04 15:44:45,217 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null")
2024-12-04 15:44:45,217 main DEBUG PluginManager 'Converter' found 47 plugins
2024-12-04 15:44:45,226 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender].
2024-12-04 15:44:45,228 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={})
2024-12-04 15:44:45,230 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR
2024-12-04 15:44:45,230 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin].
2024-12-04 15:44:45,230 main DEBUG createAppenders(={Console})
2024-12-04 15:44:45,231 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized
2024-12-04 15:44:45,231 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2
2024-12-04 15:44:45,231 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK.
2024-12-04 15:44:45,232 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1
2024-12-04 15:44:45,232 main DEBUG OutputStream closed
2024-12-04 15:44:45,232 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true
2024-12-04 15:44:45,233 main DEBUG Appender DefaultConsole-1 stopped with status true
2024-12-04 15:44:45,233 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK
2024-12-04 15:44:45,300 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6
2024-12-04 15:44:45,302 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger
2024-12-04 15:44:45,303 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector
2024-12-04 15:44:45,303 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=
2024-12-04 15:44:45,304 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory
2024-12-04 15:44:45,304 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter
2024-12-04 15:44:45,305 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper
2024-12-04 15:44:45,305 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j
2024-12-04 15:44:45,305 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl
2024-12-04 15:44:45,306 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans
2024-12-04 15:44:45,306 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase
2024-12-04 15:44:45,306 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop
2024-12-04 15:44:45,306 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers
2024-12-04 15:44:45,307 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices
2024-12-04 15:44:45,307 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig
2024-12-04 15:44:45,307 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel
2024-12-04 15:44:45,308 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore
2024-12-04 15:44:45,308 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console
2024-12-04 15:44:45,311 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-12-04 15:44:45,311 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null
2024-12-04 15:44:45,311 main DEBUG Shutdown hook enabled. Registering a new one.
2024-12-04 15:44:45,312 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK.
2024-12-04T15:44:45,326 INFO  [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins
2024-12-04 15:44:45,330 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED)
2024-12-04 15:44:45,330 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-12-04T15:44:45,606 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4
2024-12-04T15:44:45,633 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=964, ProcessCount=11, AvailableMemoryMB=10256
2024-12-04T15:44:45,740 WARN  [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
2024-12-04T15:44:45,835 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-04T15:44:45,852 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/cluster_24af270b-df31-d354-867f-9e5ac320ead1, deleteOnExit=true
2024-12-04T15:44:45,945 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/cluster_24af270b-df31-d354-867f-9e5ac320ead1/zookeeper_0, clientPort=64258, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/cluster_24af270b-df31-d354-867f-9e5ac320ead1/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/cluster_24af270b-df31-d354-867f-9e5ac320ead1/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-04T15:44:45,958 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=64258
2024-12-04T15:44:45,958 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-04T15:44:46,043 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621 with version=8
2024-12-04T15:44:46,044 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/hbase-staging
2024-12-04T15:44:46,146 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16
2024-12-04T15:44:46,416 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/a21b6491b371:0 server-side Connection retries=45
2024-12-04T15:44:46,427 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-04T15:44:46,429 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-04T15:44:46,436 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-04T15:44:46,437 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-04T15:44:46,437 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-04T15:44:46,576 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-04T15:44:46,671 WARN  [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-12-04T15:44:46,717 INFO  [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl
2024-12-04T15:44:46,731 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout
2024-12-04T15:44:46,736 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-04T15:44:46,778 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 13545 (auto-detected)
2024-12-04T15:44:46,779 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected)
2024-12-04T15:44:46,810 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43739
2024-12-04T15:44:46,833 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43739 connecting to ZooKeeper ensemble=127.0.0.1:64258
2024-12-04T15:44:46,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:437390x0, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-04T15:44:46,870 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43739-0x1017e264bf50000 connected
2024-12-04T15:44:46,899 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-04T15:44:46,900 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-04T15:44:46,908 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43739-0x1017e264bf50000, quorum=127.0.0.1:64258, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-04T15:44:46,912 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621, hbase.cluster.distributed=false
2024-12-04T15:44:46,937 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43739-0x1017e264bf50000, quorum=127.0.0.1:64258, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-04T15:44:46,943 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43739
2024-12-04T15:44:46,943 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43739
2024-12-04T15:44:46,943 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43739
2024-12-04T15:44:46,944 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43739
2024-12-04T15:44:46,944 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43739
2024-12-04T15:44:47,037 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/a21b6491b371:0 server-side Connection retries=45
2024-12-04T15:44:47,039 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-04T15:44:47,040 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-04T15:44:47,040 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-04T15:44:47,040 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-04T15:44:47,040 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-04T15:44:47,043 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-04T15:44:47,046 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-04T15:44:47,047 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46561
2024-12-04T15:44:47,049 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46561 connecting to ZooKeeper ensemble=127.0.0.1:64258
2024-12-04T15:44:47,049 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-04T15:44:47,050 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-04T15:44:47,054 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:465610x0, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-04T15:44:47,055 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46561-0x1017e264bf50001 connected
2024-12-04T15:44:47,055 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46561-0x1017e264bf50001, quorum=127.0.0.1:64258, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-04T15:44:47,061 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-04T15:44:47,073 INFO  [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/bucket.cache
2024-12-04T15:44:47,078 INFO  [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false
2024-12-04T15:44:47,079 INFO  [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread
2024-12-04T15:44:47,081 INFO  [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100
2024-12-04T15:44:47,082 INFO  [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file
2024-12-04T15:44:47,082 WARN  [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache.
2024-12-04T15:44:47,090 INFO  [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator
2024-12-04T15:44:47,094 INFO  [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size))
2024-12-04T15:44:47,094 INFO  [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/bucket.persistence is complete.
2024-12-04T15:44:47,096 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-04T15:44:47,098 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46561-0x1017e264bf50001, quorum=127.0.0.1:64258, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-04T15:44:47,105 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46561-0x1017e264bf50001, quorum=127.0.0.1:64258, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-04T15:44:47,106 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46561
2024-12-04T15:44:47,106 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46561
2024-12-04T15:44:47,107 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46561
2024-12-04T15:44:47,108 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46561
2024-12-04T15:44:47,108 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46561
2024-12-04T15:44:47,137 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/a21b6491b371:0 server-side Connection retries=45
2024-12-04T15:44:47,137 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-04T15:44:47,138 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-04T15:44:47,138 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-04T15:44:47,138 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-04T15:44:47,139 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-04T15:44:47,139 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-04T15:44:47,139 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-04T15:44:47,141 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40047
2024-12-04T15:44:47,143 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40047 connecting to ZooKeeper ensemble=127.0.0.1:64258
2024-12-04T15:44:47,144 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-04T15:44:47,145 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-04T15:44:47,150 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:400470x0, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-04T15:44:47,151 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40047-0x1017e264bf50002 connected
2024-12-04T15:44:47,151 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40047-0x1017e264bf50002, quorum=127.0.0.1:64258, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-04T15:44:47,152 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-04T15:44:47,153 INFO  [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/bucket.cache
2024-12-04T15:44:47,153 INFO  [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false
2024-12-04T15:44:47,154 INFO  [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread
2024-12-04T15:44:47,154 INFO  [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100
2024-12-04T15:44:47,157 INFO  [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file
2024-12-04T15:44:47,157 INFO  [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator
2024-12-04T15:44:47,157 WARN  [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache.
2024-12-04T15:44:47,158 INFO  [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size))
2024-12-04T15:44:47,158 INFO  [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/bucket.persistence is complete.
2024-12-04T15:44:47,158 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-04T15:44:47,160 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40047-0x1017e264bf50002, quorum=127.0.0.1:64258, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-04T15:44:47,162 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40047-0x1017e264bf50002, quorum=127.0.0.1:64258, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-04T15:44:47,165 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40047
2024-12-04T15:44:47,168 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40047
2024-12-04T15:44:47,169 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40047
2024-12-04T15:44:47,174 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40047
2024-12-04T15:44:47,174 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40047
2024-12-04T15:44:47,198 DEBUG [M:0;a21b6491b371:43739 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;a21b6491b371:43739
2024-12-04T15:44:47,200 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/a21b6491b371,43739,1733327086216
2024-12-04T15:44:47,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43739-0x1017e264bf50000, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-04T15:44:47,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46561-0x1017e264bf50001, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-04T15:44:47,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40047-0x1017e264bf50002, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-04T15:44:47,213 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43739-0x1017e264bf50000, quorum=127.0.0.1:64258, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/a21b6491b371,43739,1733327086216
2024-12-04T15:44:47,245 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46561-0x1017e264bf50001, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-04T15:44:47,245 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40047-0x1017e264bf50002, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-04T15:44:47,245 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43739-0x1017e264bf50000, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-04T15:44:47,245 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46561-0x1017e264bf50001, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-04T15:44:47,245 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40047-0x1017e264bf50002, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-04T15:44:47,248 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43739-0x1017e264bf50000, quorum=127.0.0.1:64258, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-04T15:44:47,250 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/a21b6491b371,43739,1733327086216 from backup master directory
2024-12-04T15:44:47,253 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40047-0x1017e264bf50002, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-04T15:44:47,254 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43739-0x1017e264bf50000, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/a21b6491b371,43739,1733327086216
2024-12-04T15:44:47,254 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46561-0x1017e264bf50001, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-04T15:44:47,254 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43739-0x1017e264bf50000, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-04T15:44:47,255 WARN  [master/a21b6491b371:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-04T15:44:47,256 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=a21b6491b371,43739,1733327086216
2024-12-04T15:44:47,258 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0
2024-12-04T15:44:47,260 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0
2024-12-04T15:44:47,267 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/hbase.id] with ID: cdf320bd-c43d-443c-b7bb-c132112c0f08
2024-12-04T15:44:47,267 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/.tmp/hbase.id
2024-12-04T15:44:47,274 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/hbase.id]
2024-12-04T15:44:47,317 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-04T15:44:47,322 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-04T15:44:47,343 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms.
2024-12-04T15:44:47,348 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40047-0x1017e264bf50002, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-04T15:44:47,348 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46561-0x1017e264bf50001, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-04T15:44:47,348 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43739-0x1017e264bf50000, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-04T15:44:47,377 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-04T15:44:47,381 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-04T15:44:47,430 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396
java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo)
	at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?]
	at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.<clinit>(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.lang.Class.forName0(Native Method) ~[?:?]
	at java.lang.Class.forName(Class.java:375) ~[?:?]
	at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.WALFactory.<init>(WALFactory.java:262) ~[classes/:?]
	at org.apache.hadoop.hbase.wal.WALFactory.<init>(WALFactory.java:231) ~[classes/:?]
	at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?]
	at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?]
	at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-04T15:44:47,437 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider
2024-12-04T15:44:47,497 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/MasterData/data/master/store
2024-12-04T15:44:47,519 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure.
2024-12-04T15:44:47,522 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-04T15:44:47,524 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-04T15:44:47,524 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-04T15:44:47,524 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-04T15:44:47,526 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-04T15:44:47,526 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-04T15:44:47,526 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-04T15:44:47,527 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733327087524Disabling compacts and flushes for region at 1733327087524Disabling writes for close at 1733327087526 (+2 ms)Writing region close event to WAL at 1733327087526Closed at 1733327087526
2024-12-04T15:44:47,527 WARN  [master/a21b6491b371:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/MasterData/data/master/store/.initializing
2024-12-04T15:44:47,527 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/MasterData/WALs/a21b6491b371,43739,1733327086216
2024-12-04T15:44:47,541 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName
2024-12-04T15:44:47,554 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a21b6491b371%2C43739%2C1733327086216, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/MasterData/WALs/a21b6491b371,43739,1733327086216, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/MasterData/oldWALs, maxLogs=10
2024-12-04T15:44:47,591 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/MasterData/WALs/a21b6491b371,43739,1733327086216/a21b6491b371%2C43739%2C1733327086216.1733327087559
2024-12-04T15:44:47,592 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: []
2024-12-04T15:44:47,592 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-04T15:44:47,593 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-04T15:44:47,596 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-04T15:44:47,597 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-04T15:44:47,635 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-04T15:44:47,666 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-04T15:44:47,670 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-04T15:44:47,672 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-04T15:44:47,673 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-04T15:44:47,678 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-04T15:44:47,678 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-04T15:44:47,678 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-04T15:44:47,679 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-04T15:44:47,684 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-04T15:44:47,684 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-04T15:44:47,684 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-04T15:44:47,684 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-04T15:44:47,689 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-04T15:44:47,690 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-04T15:44:47,690 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-04T15:44:47,691 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-04T15:44:47,694 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-04T15:44:47,694 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-04T15:44:47,698 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-04T15:44:47,698 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-04T15:44:47,701 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-04T15:44:47,703 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-04T15:44:47,710 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-04T15:44:47,711 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63242472, jitterRate=-0.05761373043060303}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-04T15:44:47,717 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733327087610Initializing all the Stores at 1733327087610Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733327087611 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733327087611Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733327087612 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733327087612Cleaning up temporary data from old regions at 1733327087698 (+86 ms)Region opened successfully at 1733327087717 (+19 ms)
2024-12-04T15:44:47,719 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-04T15:44:47,752 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36085e9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a21b6491b371/172.17.0.2:0
2024-12-04T15:44:47,787 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-04T15:44:47,799 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-04T15:44:47,799 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-04T15:44:47,803 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-04T15:44:47,803 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-12-04T15:44:47,809 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec
2024-12-04T15:44:47,810 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-04T15:44:47,837 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-04T15:44:47,846 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43739-0x1017e264bf50000, quorum=127.0.0.1:64258, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-04T15:44:47,850 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-04T15:44:47,854 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-04T15:44:47,856 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43739-0x1017e264bf50000, quorum=127.0.0.1:64258, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-04T15:44:47,858 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-04T15:44:47,861 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-04T15:44:47,867 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43739-0x1017e264bf50000, quorum=127.0.0.1:64258, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-04T15:44:47,869 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-04T15:44:47,871 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43739-0x1017e264bf50000, quorum=127.0.0.1:64258, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-04T15:44:47,873 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-04T15:44:47,893 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43739-0x1017e264bf50000, quorum=127.0.0.1:64258, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-04T15:44:47,896 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-04T15:44:47,909 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40047-0x1017e264bf50002, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-04T15:44:47,909 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46561-0x1017e264bf50001, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-04T15:44:47,909 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43739-0x1017e264bf50000, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-04T15:44:47,909 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43739-0x1017e264bf50000, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-04T15:44:47,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46561-0x1017e264bf50001, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-04T15:44:47,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40047-0x1017e264bf50002, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-04T15:44:47,914 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=a21b6491b371,43739,1733327086216, sessionid=0x1017e264bf50000, setting cluster-up flag (Was=false)
2024-12-04T15:44:47,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40047-0x1017e264bf50002, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-04T15:44:47,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46561-0x1017e264bf50001, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-04T15:44:47,931 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43739-0x1017e264bf50000, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-04T15:44:47,938 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-04T15:44:47,940 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a21b6491b371,43739,1733327086216
2024-12-04T15:44:47,947 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40047-0x1017e264bf50002, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-04T15:44:47,947 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46561-0x1017e264bf50001, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-04T15:44:47,947 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43739-0x1017e264bf50000, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-04T15:44:47,953 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-04T15:44:47,954 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a21b6491b371,43739,1733327086216
2024-12-04T15:44:47,959 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-04T15:44:47,978 INFO  [RS:0;a21b6491b371:46561 {}] regionserver.HRegionServer(746): ClusterId : cdf320bd-c43d-443c-b7bb-c132112c0f08
2024-12-04T15:44:47,978 INFO  [RS:1;a21b6491b371:40047 {}] regionserver.HRegionServer(746): ClusterId : cdf320bd-c43d-443c-b7bb-c132112c0f08
2024-12-04T15:44:47,981 DEBUG [RS:1;a21b6491b371:40047 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-04T15:44:47,981 DEBUG [RS:0;a21b6491b371:46561 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-04T15:44:47,987 DEBUG [RS:0;a21b6491b371:46561 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-04T15:44:47,987 DEBUG [RS:1;a21b6491b371:40047 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-04T15:44:47,987 DEBUG [RS:0;a21b6491b371:46561 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-04T15:44:47,987 DEBUG [RS:1;a21b6491b371:40047 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-04T15:44:47,991 DEBUG [RS:1;a21b6491b371:40047 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-04T15:44:47,991 DEBUG [RS:0;a21b6491b371:46561 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-04T15:44:47,992 DEBUG [RS:1;a21b6491b371:40047 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65936eb6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a21b6491b371/172.17.0.2:0
2024-12-04T15:44:47,992 DEBUG [RS:0;a21b6491b371:46561 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c3d50a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a21b6491b371/172.17.0.2:0
2024-12-04T15:44:48,010 DEBUG [RS:0;a21b6491b371:46561 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;a21b6491b371:46561
2024-12-04T15:44:48,011 DEBUG [RS:1;a21b6491b371:40047 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;a21b6491b371:40047
2024-12-04T15:44:48,013 INFO  [RS:0;a21b6491b371:46561 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-04T15:44:48,013 INFO  [RS:1;a21b6491b371:40047 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-04T15:44:48,013 INFO  [RS:0;a21b6491b371:46561 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-04T15:44:48,013 INFO  [RS:1;a21b6491b371:40047 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-04T15:44:48,013 DEBUG [RS:0;a21b6491b371:46561 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-04T15:44:48,013 DEBUG [RS:1;a21b6491b371:40047 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-04T15:44:48,016 INFO  [RS:0;a21b6491b371:46561 {}] regionserver.HRegionServer(2659): reportForDuty to master=a21b6491b371,43739,1733327086216 with port=46561, startcode=1733327086997
2024-12-04T15:44:48,016 INFO  [RS:1;a21b6491b371:40047 {}] regionserver.HRegionServer(2659): reportForDuty to master=a21b6491b371,43739,1733327086216 with port=40047, startcode=1733327087136
2024-12-04T15:44:48,029 DEBUG [RS:1;a21b6491b371:40047 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-04T15:44:48,029 DEBUG [RS:0;a21b6491b371:46561 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-04T15:44:48,038 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-04T15:44:48,050 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-04T15:44:48,058 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-04T15:44:48,064 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: a21b6491b371,43739,1733327086216
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-04T15:44:48,072 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/a21b6491b371:0, corePoolSize=5, maxPoolSize=5
2024-12-04T15:44:48,073 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/a21b6491b371:0, corePoolSize=5, maxPoolSize=5
2024-12-04T15:44:48,075 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/a21b6491b371:0, corePoolSize=5, maxPoolSize=5
2024-12-04T15:44:48,078 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/a21b6491b371:0, corePoolSize=5, maxPoolSize=5
2024-12-04T15:44:48,078 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/a21b6491b371:0, corePoolSize=10, maxPoolSize=10
2024-12-04T15:44:48,078 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:48,078 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/a21b6491b371:0, corePoolSize=2, maxPoolSize=2
2024-12-04T15:44:48,079 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:48,083 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-04T15:44:48,084 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-04T15:44:48,088 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-04T15:44:48,089 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733327118089
2024-12-04T15:44:48,088 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-04T15:44:48,091 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-04T15:44:48,092 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-04T15:44:48,095 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-04T15:44:48,095 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-04T15:44:48,096 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-04T15:44:48,096 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-04T15:44:48,111 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50207, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService
2024-12-04T15:44:48,111 INFO  [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44091, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService
2024-12-04T15:44:48,108 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:48,113 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-04T15:44:48,115 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-04T15:44:48,115 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-04T15:44:48,115 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-04T15:44:48,116 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621
2024-12-04T15:44:48,120 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-04T15:44:48,120 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43739 {}] ipc.MetricsHBaseServer(152): Unknown exception type
org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet
	at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?]
	at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
2024-12-04T15:44:48,121 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-04T15:44:48,124 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/a21b6491b371:0:becomeActiveMaster-HFileCleaner.large.0-1733327088122,5,FailOnTimeoutGroup]
2024-12-04T15:44:48,128 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43739 {}] ipc.MetricsHBaseServer(152): Unknown exception type
org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet
	at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?]
	at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT]
2024-12-04T15:44:48,129 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/a21b6491b371:0:becomeActiveMaster-HFileCleaner.small.0-1733327088125,5,FailOnTimeoutGroup]
2024-12-04T15:44:48,129 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:48,130 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-04T15:44:48,131 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-04T15:44:48,131 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:48,131 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:48,133 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 
2024-12-04T15:44:48,146 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-04T15:44:48,146 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-04T15:44:48,146 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-04T15:44:48,147 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 
2024-12-04T15:44:48,152 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-04T15:44:48,152 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-04T15:44:48,152 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-04T15:44:48,153 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 
2024-12-04T15:44:48,155 DEBUG [RS:1;a21b6491b371:40047 {}] regionserver.HRegionServer(2683): Master is not running yet
2024-12-04T15:44:48,155 DEBUG [RS:0;a21b6491b371:46561 {}] regionserver.HRegionServer(2683): Master is not running yet
2024-12-04T15:44:48,155 WARN  [RS:1;a21b6491b371:40047 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying.
2024-12-04T15:44:48,155 WARN  [RS:0;a21b6491b371:46561 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying.
2024-12-04T15:44:48,158 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-04T15:44:48,158 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-04T15:44:48,159 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-04T15:44:48,159 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 
2024-12-04T15:44:48,164 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-04T15:44:48,164 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-04T15:44:48,164 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-04T15:44:48,165 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-04T15:44:48,166 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740
2024-12-04T15:44:48,167 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740
2024-12-04T15:44:48,167 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-04T15:44:48,167 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-04T15:44:48,168 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-04T15:44:48,169 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-04T15:44:48,177 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-04T15:44:48,177 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63057918, jitterRate=-0.06036379933357239}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-04T15:44:48,180 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733327088131Initializing all the Stores at 1733327088131Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733327088132 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733327088133 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733327088133Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733327088133Cleaning up temporary data from old regions at 1733327088168 (+35 ms)Region opened successfully at 1733327088180 (+12 ms)
2024-12-04T15:44:48,181 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-04T15:44:48,181 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-04T15:44:48,181 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-04T15:44:48,181 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-04T15:44:48,181 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-04T15:44:48,182 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-04T15:44:48,183 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733327088181Disabling compacts and flushes for region at 1733327088181Disabling writes for close at 1733327088181Writing region close event to WAL at 1733327088182 (+1 ms)Closed at 1733327088182
2024-12-04T15:44:48,185 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-04T15:44:48,185 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-04T15:44:48,192 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-04T15:44:48,200 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-04T15:44:48,202 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-04T15:44:48,256 INFO  [RS:0;a21b6491b371:46561 {}] regionserver.HRegionServer(2659): reportForDuty to master=a21b6491b371,43739,1733327086216 with port=46561, startcode=1733327086997
2024-12-04T15:44:48,256 INFO  [RS:1;a21b6491b371:40047 {}] regionserver.HRegionServer(2659): reportForDuty to master=a21b6491b371,43739,1733327086216 with port=40047, startcode=1733327087136
2024-12-04T15:44:48,258 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43739 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a21b6491b371,46561,1733327086997
2024-12-04T15:44:48,261 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43739 {}] master.ServerManager(517): Registering regionserver=a21b6491b371,46561,1733327086997
2024-12-04T15:44:48,268 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43739 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a21b6491b371,40047,1733327087136
2024-12-04T15:44:48,268 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43739 {}] master.ServerManager(517): Registering regionserver=a21b6491b371,40047,1733327087136
2024-12-04T15:44:48,269 DEBUG [RS:0;a21b6491b371:46561 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621
2024-12-04T15:44:48,269 DEBUG [RS:0;a21b6491b371:46561 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/
2024-12-04T15:44:48,269 DEBUG [RS:0;a21b6491b371:46561 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-04T15:44:48,271 DEBUG [RS:1;a21b6491b371:40047 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621
2024-12-04T15:44:48,272 DEBUG [RS:1;a21b6491b371:40047 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/
2024-12-04T15:44:48,272 DEBUG [RS:1;a21b6491b371:40047 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-04T15:44:48,283 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43739-0x1017e264bf50000, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-04T15:44:48,284 DEBUG [RS:0;a21b6491b371:46561 {}] zookeeper.ZKUtil(111): regionserver:46561-0x1017e264bf50001, quorum=127.0.0.1:64258, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a21b6491b371,46561,1733327086997
2024-12-04T15:44:48,284 WARN  [RS:0;a21b6491b371:46561 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-04T15:44:48,284 DEBUG [RS:1;a21b6491b371:40047 {}] zookeeper.ZKUtil(111): regionserver:40047-0x1017e264bf50002, quorum=127.0.0.1:64258, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a21b6491b371,40047,1733327087136
2024-12-04T15:44:48,284 WARN  [RS:1;a21b6491b371:40047 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-04T15:44:48,285 INFO  [RS:0;a21b6491b371:46561 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider
2024-12-04T15:44:48,285 INFO  [RS:1;a21b6491b371:40047 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider
2024-12-04T15:44:48,285 DEBUG [RS:0;a21b6491b371:46561 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/WALs/a21b6491b371,46561,1733327086997
2024-12-04T15:44:48,285 DEBUG [RS:1;a21b6491b371:40047 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/WALs/a21b6491b371,40047,1733327087136
2024-12-04T15:44:48,286 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a21b6491b371,40047,1733327087136]
2024-12-04T15:44:48,286 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a21b6491b371,46561,1733327086997]
2024-12-04T15:44:48,311 INFO  [RS:1;a21b6491b371:40047 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-04T15:44:48,311 INFO  [RS:0;a21b6491b371:46561 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-04T15:44:48,328 INFO  [RS:1;a21b6491b371:40047 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-04T15:44:48,328 INFO  [RS:0;a21b6491b371:46561 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-04T15:44:48,333 INFO  [RS:0;a21b6491b371:46561 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-04T15:44:48,333 INFO  [RS:1;a21b6491b371:40047 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-04T15:44:48,333 INFO  [RS:1;a21b6491b371:40047 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:48,333 INFO  [RS:0;a21b6491b371:46561 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:48,334 INFO  [RS:0;a21b6491b371:46561 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-04T15:44:48,334 INFO  [RS:1;a21b6491b371:40047 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-04T15:44:48,340 INFO  [RS:0;a21b6491b371:46561 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-04T15:44:48,340 INFO  [RS:1;a21b6491b371:40047 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-04T15:44:48,342 INFO  [RS:1;a21b6491b371:40047 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:48,342 INFO  [RS:0;a21b6491b371:46561 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:48,342 DEBUG [RS:1;a21b6491b371:40047 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:48,342 DEBUG [RS:0;a21b6491b371:46561 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:48,342 DEBUG [RS:1;a21b6491b371:40047 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:48,342 DEBUG [RS:0;a21b6491b371:46561 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:48,343 DEBUG [RS:1;a21b6491b371:40047 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:48,343 DEBUG [RS:0;a21b6491b371:46561 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:48,343 DEBUG [RS:0;a21b6491b371:46561 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:48,343 DEBUG [RS:1;a21b6491b371:40047 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:48,343 DEBUG [RS:0;a21b6491b371:46561 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:48,343 DEBUG [RS:1;a21b6491b371:40047 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:48,343 DEBUG [RS:0;a21b6491b371:46561 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a21b6491b371:0, corePoolSize=2, maxPoolSize=2
2024-12-04T15:44:48,343 DEBUG [RS:1;a21b6491b371:40047 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a21b6491b371:0, corePoolSize=2, maxPoolSize=2
2024-12-04T15:44:48,343 DEBUG [RS:1;a21b6491b371:40047 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:48,343 DEBUG [RS:0;a21b6491b371:46561 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:48,344 DEBUG [RS:0;a21b6491b371:46561 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:48,344 DEBUG [RS:1;a21b6491b371:40047 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:48,344 DEBUG [RS:0;a21b6491b371:46561 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:48,344 DEBUG [RS:1;a21b6491b371:40047 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:48,344 DEBUG [RS:1;a21b6491b371:40047 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:48,344 DEBUG [RS:0;a21b6491b371:46561 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:48,344 DEBUG [RS:0;a21b6491b371:46561 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:48,344 DEBUG [RS:1;a21b6491b371:40047 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:48,345 DEBUG [RS:0;a21b6491b371:46561 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:48,345 DEBUG [RS:1;a21b6491b371:40047 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:48,345 DEBUG [RS:1;a21b6491b371:40047 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a21b6491b371:0, corePoolSize=3, maxPoolSize=3
2024-12-04T15:44:48,345 DEBUG [RS:0;a21b6491b371:46561 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a21b6491b371:0, corePoolSize=3, maxPoolSize=3
2024-12-04T15:44:48,345 DEBUG [RS:0;a21b6491b371:46561 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a21b6491b371:0, corePoolSize=3, maxPoolSize=3
2024-12-04T15:44:48,345 DEBUG [RS:1;a21b6491b371:40047 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a21b6491b371:0, corePoolSize=3, maxPoolSize=3
2024-12-04T15:44:48,351 INFO  [RS:0;a21b6491b371:46561 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:48,351 INFO  [RS:0;a21b6491b371:46561 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:48,352 INFO  [RS:0;a21b6491b371:46561 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:48,352 INFO  [RS:0;a21b6491b371:46561 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:48,352 INFO  [RS:0;a21b6491b371:46561 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:48,352 INFO  [RS:0;a21b6491b371:46561 {}] hbase.ChoreService(168): Chore ScheduledChore name=a21b6491b371,46561,1733327086997-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-04T15:44:48,353 INFO  [RS:1;a21b6491b371:40047 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:48,353 INFO  [RS:1;a21b6491b371:40047 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:48,353 WARN  [a21b6491b371:43739 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-12-04T15:44:48,353 INFO  [RS:1;a21b6491b371:40047 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:48,354 INFO  [RS:1;a21b6491b371:40047 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:48,354 INFO  [RS:1;a21b6491b371:40047 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:48,354 INFO  [RS:1;a21b6491b371:40047 {}] hbase.ChoreService(168): Chore ScheduledChore name=a21b6491b371,40047,1733327087136-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-04T15:44:48,377 INFO  [RS:1;a21b6491b371:40047 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-04T15:44:48,378 INFO  [RS:0;a21b6491b371:46561 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-04T15:44:48,379 INFO  [RS:1;a21b6491b371:40047 {}] hbase.ChoreService(168): Chore ScheduledChore name=a21b6491b371,40047,1733327087136-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:48,380 INFO  [RS:0;a21b6491b371:46561 {}] hbase.ChoreService(168): Chore ScheduledChore name=a21b6491b371,46561,1733327086997-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:48,380 INFO  [RS:1;a21b6491b371:40047 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:48,380 INFO  [RS:0;a21b6491b371:46561 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:48,380 INFO  [RS:1;a21b6491b371:40047 {}] regionserver.Replication(171): a21b6491b371,40047,1733327087136 started
2024-12-04T15:44:48,380 INFO  [RS:0;a21b6491b371:46561 {}] regionserver.Replication(171): a21b6491b371,46561,1733327086997 started
2024-12-04T15:44:48,400 INFO  [RS:1;a21b6491b371:40047 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:48,401 INFO  [RS:1;a21b6491b371:40047 {}] regionserver.HRegionServer(1482): Serving as a21b6491b371,40047,1733327087136, RpcServer on a21b6491b371/172.17.0.2:40047, sessionid=0x1017e264bf50002
2024-12-04T15:44:48,402 DEBUG [RS:1;a21b6491b371:40047 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-04T15:44:48,402 DEBUG [RS:1;a21b6491b371:40047 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a21b6491b371,40047,1733327087136
2024-12-04T15:44:48,402 DEBUG [RS:1;a21b6491b371:40047 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a21b6491b371,40047,1733327087136'
2024-12-04T15:44:48,402 DEBUG [RS:1;a21b6491b371:40047 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-04T15:44:48,403 DEBUG [RS:1;a21b6491b371:40047 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-04T15:44:48,404 DEBUG [RS:1;a21b6491b371:40047 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-04T15:44:48,404 DEBUG [RS:1;a21b6491b371:40047 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-04T15:44:48,404 DEBUG [RS:1;a21b6491b371:40047 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a21b6491b371,40047,1733327087136
2024-12-04T15:44:48,404 DEBUG [RS:1;a21b6491b371:40047 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a21b6491b371,40047,1733327087136'
2024-12-04T15:44:48,405 DEBUG [RS:1;a21b6491b371:40047 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-04T15:44:48,405 INFO  [RS:0;a21b6491b371:46561 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:48,405 INFO  [RS:0;a21b6491b371:46561 {}] regionserver.HRegionServer(1482): Serving as a21b6491b371,46561,1733327086997, RpcServer on a21b6491b371/172.17.0.2:46561, sessionid=0x1017e264bf50001
2024-12-04T15:44:48,405 DEBUG [RS:1;a21b6491b371:40047 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-04T15:44:48,406 DEBUG [RS:0;a21b6491b371:46561 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-04T15:44:48,406 DEBUG [RS:0;a21b6491b371:46561 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a21b6491b371,46561,1733327086997
2024-12-04T15:44:48,406 DEBUG [RS:0;a21b6491b371:46561 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a21b6491b371,46561,1733327086997'
2024-12-04T15:44:48,406 DEBUG [RS:0;a21b6491b371:46561 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-04T15:44:48,406 DEBUG [RS:1;a21b6491b371:40047 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-04T15:44:48,406 INFO  [RS:1;a21b6491b371:40047 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-04T15:44:48,406 INFO  [RS:1;a21b6491b371:40047 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-04T15:44:48,407 DEBUG [RS:0;a21b6491b371:46561 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-04T15:44:48,407 DEBUG [RS:0;a21b6491b371:46561 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-04T15:44:48,408 DEBUG [RS:0;a21b6491b371:46561 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-04T15:44:48,408 DEBUG [RS:0;a21b6491b371:46561 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a21b6491b371,46561,1733327086997
2024-12-04T15:44:48,408 DEBUG [RS:0;a21b6491b371:46561 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a21b6491b371,46561,1733327086997'
2024-12-04T15:44:48,408 DEBUG [RS:0;a21b6491b371:46561 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-04T15:44:48,409 DEBUG [RS:0;a21b6491b371:46561 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-04T15:44:48,410 DEBUG [RS:0;a21b6491b371:46561 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-04T15:44:48,410 INFO  [RS:0;a21b6491b371:46561 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-04T15:44:48,410 INFO  [RS:0;a21b6491b371:46561 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-04T15:44:48,513 INFO  [RS:1;a21b6491b371:40047 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName
2024-12-04T15:44:48,513 INFO  [RS:0;a21b6491b371:46561 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName
2024-12-04T15:44:48,514 INFO  [RS:1;a21b6491b371:40047 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=a21b6491b371%2C40047%2C1733327087136, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/WALs/a21b6491b371,40047,1733327087136, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/oldWALs, maxLogs=55
2024-12-04T15:44:48,514 INFO  [RS:0;a21b6491b371:46561 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=a21b6491b371%2C46561%2C1733327086997, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/WALs/a21b6491b371,46561,1733327086997, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/oldWALs, maxLogs=55
2024-12-04T15:44:48,546 INFO  [RS:0;a21b6491b371:46561 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/WALs/a21b6491b371,46561,1733327086997/a21b6491b371%2C46561%2C1733327086997.1733327088519
2024-12-04T15:44:48,546 INFO  [RS:1;a21b6491b371:40047 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/WALs/a21b6491b371,40047,1733327087136/a21b6491b371%2C40047%2C1733327087136.1733327088519
2024-12-04T15:44:48,546 DEBUG [RS:1;a21b6491b371:40047 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: []
2024-12-04T15:44:48,546 DEBUG [RS:0;a21b6491b371:46561 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: []
2024-12-04T15:44:48,606 DEBUG [a21b6491b371:43739 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2
2024-12-04T15:44:48,618 DEBUG [a21b6491b371:43739 {}] balancer.BalancerClusterState(204): Hosts are {a21b6491b371=0} racks are {/default-rack=0}
2024-12-04T15:44:48,622 DEBUG [a21b6491b371:43739 {}] balancer.BalancerClusterState(303): server 0 has 0 regions
2024-12-04T15:44:48,622 DEBUG [a21b6491b371:43739 {}] balancer.BalancerClusterState(303): server 1 has 0 regions
2024-12-04T15:44:48,622 DEBUG [a21b6491b371:43739 {}] balancer.BalancerClusterState(310): server 0 is on host 0
2024-12-04T15:44:48,622 DEBUG [a21b6491b371:43739 {}] balancer.BalancerClusterState(310): server 1 is on host 0
2024-12-04T15:44:48,622 INFO  [a21b6491b371:43739 {}] balancer.BalancerClusterState(321): server 0 is on rack 0
2024-12-04T15:44:48,622 INFO  [a21b6491b371:43739 {}] balancer.BalancerClusterState(321): server 1 is on rack 0
2024-12-04T15:44:48,622 DEBUG [a21b6491b371:43739 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1
2024-12-04T15:44:48,631 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a21b6491b371,40047,1733327087136
2024-12-04T15:44:48,637 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a21b6491b371,40047,1733327087136, state=OPENING
2024-12-04T15:44:48,644 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-04T15:44:48,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40047-0x1017e264bf50002, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-04T15:44:48,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46561-0x1017e264bf50001, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-04T15:44:48,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43739-0x1017e264bf50000, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-04T15:44:48,647 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-04T15:44:48,647 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-04T15:44:48,647 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-04T15:44:48,649 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-04T15:44:48,651 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a21b6491b371,40047,1733327087136}]
2024-12-04T15:44:48,824 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-04T15:44:48,828 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57277, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-04T15:44:48,837 INFO  [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-04T15:44:48,837 INFO  [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider
2024-12-04T15:44:48,838 INFO  [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta
2024-12-04T15:44:48,839 INFO  [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=a21b6491b371%2C40047%2C1733327087136.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/WALs/a21b6491b371,40047,1733327087136, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/oldWALs, maxLogs=55
2024-12-04T15:44:48,860 INFO  [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/WALs/a21b6491b371,40047,1733327087136/a21b6491b371%2C40047%2C1733327087136.meta.1733327088841.meta
2024-12-04T15:44:48,861 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: []
2024-12-04T15:44:48,861 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-04T15:44:48,863 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-04T15:44:48,866 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-04T15:44:48,870 INFO  [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-04T15:44:48,875 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-04T15:44:48,875 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-04T15:44:48,876 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-04T15:44:48,876 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-04T15:44:48,877 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 
2024-12-04T15:44:48,878 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-04T15:44:48,878 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-04T15:44:48,879 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-04T15:44:48,879 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 
2024-12-04T15:44:48,880 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-04T15:44:48,880 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-04T15:44:48,880 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-04T15:44:48,880 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 
2024-12-04T15:44:48,881 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-04T15:44:48,881 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-04T15:44:48,881 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-04T15:44:48,881 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 
2024-12-04T15:44:48,882 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-04T15:44:48,882 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-04T15:44:48,882 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-04T15:44:48,883 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-04T15:44:48,884 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740
2024-12-04T15:44:48,884 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740
2024-12-04T15:44:48,885 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-04T15:44:48,885 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-04T15:44:48,885 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-04T15:44:48,886 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-04T15:44:48,887 INFO  [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74104197, jitterRate=0.1042385846376419}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-04T15:44:48,887 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-04T15:44:48,889 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733327088876Writing region info on filesystem at 1733327088877 (+1 ms)Initializing all the Stores at 1733327088877Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733327088877Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733327088877Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733327088877Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733327088877Cleaning up temporary data from old regions at 1733327088885 (+8 ms)Running coprocessor post-open hooks at 1733327088887 (+2 ms)Region opened successfully at 1733327088888 (+1 ms)
2024-12-04T15:44:48,894 INFO  [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733327088816
2024-12-04T15:44:48,905 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-04T15:44:48,905 INFO  [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-04T15:44:48,906 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=a21b6491b371,40047,1733327087136
2024-12-04T15:44:48,907 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a21b6491b371,40047,1733327087136, state=OPEN
2024-12-04T15:44:48,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43739-0x1017e264bf50000, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-04T15:44:48,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40047-0x1017e264bf50002, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-04T15:44:48,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46561-0x1017e264bf50001, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-04T15:44:48,910 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-04T15:44:48,910 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-04T15:44:48,910 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-04T15:44:48,911 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a21b6491b371,40047,1733327087136
2024-12-04T15:44:48,914 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-04T15:44:48,915 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a21b6491b371,40047,1733327087136 in 260 msec
2024-12-04T15:44:48,919 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-04T15:44:48,919 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 724 msec
2024-12-04T15:44:48,920 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-04T15:44:48,920 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-04T15:44:48,942 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-04T15:44:48,944 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a21b6491b371,40047,1733327087136, seqNum=-1]
2024-12-04T15:44:48,971 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-04T15:44:48,974 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46793, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-04T15:44:48,995 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0060 sec
2024-12-04T15:44:48,995 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733327088995, completionTime=-1
2024-12-04T15:44:48,998 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running
2024-12-04T15:44:48,998 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-04T15:44:49,023 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2
2024-12-04T15:44:49,024 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733327149024
2024-12-04T15:44:49,024 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733327209024
2024-12-04T15:44:49,024 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 25 msec
2024-12-04T15:44:49,026 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache
2024-12-04T15:44:49,033 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a21b6491b371,43739,1733327086216-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:49,034 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a21b6491b371,43739,1733327086216-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:49,034 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a21b6491b371,43739,1733327086216-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:49,035 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-a21b6491b371:43739, period=300000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:49,036 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:49,036 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:49,042 DEBUG [master/a21b6491b371:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-04T15:44:49,068 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.812sec
2024-12-04T15:44:49,070 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-04T15:44:49,071 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-04T15:44:49,072 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-04T15:44:49,072 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-04T15:44:49,072 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-04T15:44:49,073 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a21b6491b371,43739,1733327086216-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-04T15:44:49,074 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a21b6491b371,43739,1733327086216-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-04T15:44:49,075 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-04T15:44:49,076 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-04T15:44:49,076 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a21b6491b371,43739,1733327086216-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:49,088 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c7dcd51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-04T15:44:49,091 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false
2024-12-04T15:44:49,091 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512
2024-12-04T15:44:49,095 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a21b6491b371,43739,-1 for getting cluster id
2024-12-04T15:44:49,098 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-04T15:44:49,106 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cdf320bd-c43d-443c-b7bb-c132112c0f08'
2024-12-04T15:44:49,109 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-04T15:44:49,109 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cdf320bd-c43d-443c-b7bb-c132112c0f08"

2024-12-04T15:44:49,109 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4367b242, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-04T15:44:49,110 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a21b6491b371,43739,-1]
2024-12-04T15:44:49,112 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-04T15:44:49,114 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-04T15:44:49,115 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59282, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-04T15:44:49,118 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b7a9956, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-04T15:44:49,118 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-04T15:44:49,127 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a21b6491b371,40047,1733327087136, seqNum=-1]
2024-12-04T15:44:49,127 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-04T15:44:49,130 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49068, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-04T15:44:49,160 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=a21b6491b371,43739,1733327086216
2024-12-04T15:44:49,166 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-12-04T15:44:49,171 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is a21b6491b371,43739,1733327086216
2024-12-04T15:44:49,174 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6399272d
2024-12-04T15:44:49,177 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-04T15:44:49,180 INFO  [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59290, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-04T15:44:49,186 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43739 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1733327089160', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-04T15:44:49,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43739 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1733327089160
2024-12-04T15:44:49,196 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733327089160 execute state=CREATE_TABLE_PRE_OPERATION
2024-12-04T15:44:49,198 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43739 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "testBlockEvictionOnRegionMove1733327089160"
 procId is: 4
2024-12-04T15:44:49,198 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-04T15:44:49,200 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733327089160 execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-04T15:44:49,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-04T15:44:49,214 INFO  [RegionOpenAndInit-testBlockEvictionOnRegionMove1733327089160-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2d7a4349fa18473dcc1fbca94949bd61, NAME => 'testBlockEvictionOnRegionMove1733327089160,,1733327089182.2d7a4349fa18473dcc1fbca94949bd61.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1733327089160', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621
2024-12-04T15:44:49,221 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733327089160-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733327089160,,1733327089182.2d7a4349fa18473dcc1fbca94949bd61.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-04T15:44:49,222 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733327089160-pool-0 {}] regionserver.HRegion(1722): Closing 2d7a4349fa18473dcc1fbca94949bd61, disabling compactions & flushes
2024-12-04T15:44:49,222 INFO  [RegionOpenAndInit-testBlockEvictionOnRegionMove1733327089160-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733327089160,,1733327089182.2d7a4349fa18473dcc1fbca94949bd61.
2024-12-04T15:44:49,222 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733327089160-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733327089160,,1733327089182.2d7a4349fa18473dcc1fbca94949bd61.
2024-12-04T15:44:49,222 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733327089160-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733327089160,,1733327089182.2d7a4349fa18473dcc1fbca94949bd61. after waiting 0 ms
2024-12-04T15:44:49,222 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733327089160-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733327089160,,1733327089182.2d7a4349fa18473dcc1fbca94949bd61.
2024-12-04T15:44:49,222 INFO  [RegionOpenAndInit-testBlockEvictionOnRegionMove1733327089160-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733327089160,,1733327089182.2d7a4349fa18473dcc1fbca94949bd61.
2024-12-04T15:44:49,222 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733327089160-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2d7a4349fa18473dcc1fbca94949bd61:
Waiting for close lock at 1733327089221Disabling compacts and flushes for region at 1733327089221Disabling writes for close at 1733327089222 (+1 ms)Writing region close event to WAL at 1733327089222Closed at 1733327089222
2024-12-04T15:44:49,223 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733327089160 execute state=CREATE_TABLE_ADD_TO_META
2024-12-04T15:44:49,229 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1733327089160,,1733327089182.2d7a4349fa18473dcc1fbca94949bd61.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733327089224"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733327089224"}]},"ts":"1733327089224"}
2024-12-04T15:44:49,233 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-12-04T15:44:49,235 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733327089160 execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-04T15:44:49,237 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1733327089160","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733327089235"}]},"ts":"1733327089235"}
2024-12-04T15:44:49,241 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1733327089160, state=ENABLING in hbase:meta
2024-12-04T15:44:49,242 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {a21b6491b371=0} racks are {/default-rack=0}
2024-12-04T15:44:49,243 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions
2024-12-04T15:44:49,243 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions
2024-12-04T15:44:49,243 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0
2024-12-04T15:44:49,243 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0
2024-12-04T15:44:49,243 INFO  [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0
2024-12-04T15:44:49,243 INFO  [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0
2024-12-04T15:44:49,243 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1
2024-12-04T15:44:49,245 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733327089160, region=2d7a4349fa18473dcc1fbca94949bd61, ASSIGN}]
2024-12-04T15:44:49,246 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733327089160, region=2d7a4349fa18473dcc1fbca94949bd61, ASSIGN
2024-12-04T15:44:49,247 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733327089160, region=2d7a4349fa18473dcc1fbca94949bd61, ASSIGN; state=OFFLINE, location=a21b6491b371,40047,1733327087136; forceNewPlan=false, retain=false
2024-12-04T15:44:49,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-04T15:44:49,400 INFO  [a21b6491b371:43739 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 
2024-12-04T15:44:49,401 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=2d7a4349fa18473dcc1fbca94949bd61, regionState=OPENING, regionLocation=a21b6491b371,40047,1733327087136
2024-12-04T15:44:49,404 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733327089160, region=2d7a4349fa18473dcc1fbca94949bd61, ASSIGN because future has completed
2024-12-04T15:44:49,405 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2d7a4349fa18473dcc1fbca94949bd61, server=a21b6491b371,40047,1733327087136}]
2024-12-04T15:44:49,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-04T15:44:49,561 INFO  [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1733327089160,,1733327089182.2d7a4349fa18473dcc1fbca94949bd61.
2024-12-04T15:44:49,562 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 2d7a4349fa18473dcc1fbca94949bd61, NAME => 'testBlockEvictionOnRegionMove1733327089160,,1733327089182.2d7a4349fa18473dcc1fbca94949bd61.', STARTKEY => '', ENDKEY => ''}
2024-12-04T15:44:49,562 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1733327089160 2d7a4349fa18473dcc1fbca94949bd61
2024-12-04T15:44:49,562 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733327089160,,1733327089182.2d7a4349fa18473dcc1fbca94949bd61.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-04T15:44:49,563 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 2d7a4349fa18473dcc1fbca94949bd61
2024-12-04T15:44:49,563 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 2d7a4349fa18473dcc1fbca94949bd61
2024-12-04T15:44:49,564 INFO  [StoreOpener-2d7a4349fa18473dcc1fbca94949bd61-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 2d7a4349fa18473dcc1fbca94949bd61 
2024-12-04T15:44:49,568 INFO  [StoreOpener-2d7a4349fa18473dcc1fbca94949bd61-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d7a4349fa18473dcc1fbca94949bd61 columnFamilyName family
2024-12-04T15:44:49,568 DEBUG [StoreOpener-2d7a4349fa18473dcc1fbca94949bd61-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-04T15:44:49,569 INFO  [StoreOpener-2d7a4349fa18473dcc1fbca94949bd61-1 {}] regionserver.HStore(327): Store=2d7a4349fa18473dcc1fbca94949bd61/family,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-04T15:44:49,569 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 2d7a4349fa18473dcc1fbca94949bd61
2024-12-04T15:44:49,570 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/default/testBlockEvictionOnRegionMove1733327089160/2d7a4349fa18473dcc1fbca94949bd61
2024-12-04T15:44:49,570 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/default/testBlockEvictionOnRegionMove1733327089160/2d7a4349fa18473dcc1fbca94949bd61
2024-12-04T15:44:49,571 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 2d7a4349fa18473dcc1fbca94949bd61
2024-12-04T15:44:49,571 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 2d7a4349fa18473dcc1fbca94949bd61
2024-12-04T15:44:49,572 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 2d7a4349fa18473dcc1fbca94949bd61
2024-12-04T15:44:49,578 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/default/testBlockEvictionOnRegionMove1733327089160/2d7a4349fa18473dcc1fbca94949bd61/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-04T15:44:49,579 INFO  [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 2d7a4349fa18473dcc1fbca94949bd61; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69296182, jitterRate=0.03259357810020447}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-04T15:44:49,579 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2d7a4349fa18473dcc1fbca94949bd61
2024-12-04T15:44:49,580 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 2d7a4349fa18473dcc1fbca94949bd61:
Running coprocessor pre-open hook at 1733327089563Writing region info on filesystem at 1733327089563Initializing all the Stores at 1733327089563Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733327089563Cleaning up temporary data from old regions at 1733327089571 (+8 ms)Running coprocessor post-open hooks at 1733327089579 (+8 ms)Region opened successfully at 1733327089580 (+1 ms)
2024-12-04T15:44:49,581 INFO  [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1733327089160,,1733327089182.2d7a4349fa18473dcc1fbca94949bd61., pid=6, masterSystemTime=1733327089558
2024-12-04T15:44:49,585 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1733327089160,,1733327089182.2d7a4349fa18473dcc1fbca94949bd61.
2024-12-04T15:44:49,585 INFO  [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1733327089160,,1733327089182.2d7a4349fa18473dcc1fbca94949bd61.
2024-12-04T15:44:49,585 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=2d7a4349fa18473dcc1fbca94949bd61, regionState=OPEN, openSeqNum=2, regionLocation=a21b6491b371,40047,1733327087136
2024-12-04T15:44:49,589 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2d7a4349fa18473dcc1fbca94949bd61, server=a21b6491b371,40047,1733327087136 because future has completed
2024-12-04T15:44:49,592 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-12-04T15:44:49,592 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 2d7a4349fa18473dcc1fbca94949bd61, server=a21b6491b371,40047,1733327087136 in 185 msec
2024-12-04T15:44:49,595 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-12-04T15:44:49,595 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733327089160, region=2d7a4349fa18473dcc1fbca94949bd61, ASSIGN in 348 msec
2024-12-04T15:44:49,596 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733327089160 execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-04T15:44:49,596 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1733327089160","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733327089596"}]},"ts":"1733327089596"}
2024-12-04T15:44:49,599 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1733327089160, state=ENABLED in hbase:meta
2024-12-04T15:44:49,600 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733327089160 execute state=CREATE_TABLE_POST_OPERATION
2024-12-04T15:44:49,602 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1733327089160 in 411 msec
2024-12-04T15:44:49,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43739 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-04T15:44:49,834 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1733327089160 get assigned. Timeout = 60000ms
2024-12-04T15:44:49,834 INFO  [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1733327089160 completed
2024-12-04T15:44:49,837 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1])
2024-12-04T15:44:49,842 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1733327089160 assigned to meta. Checking AM states.
2024-12-04T15:44:49,842 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1])
2024-12-04T15:44:49,843 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1733327089160 assigned.
2024-12-04T15:44:49,851 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1733327089160', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1733327089160,,1733327089182.2d7a4349fa18473dcc1fbca94949bd61., hostname=a21b6491b371,40047,1733327087136, seqNum=2]
2024-12-04T15:44:49,865 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 2d7a4349fa18473dcc1fbca94949bd61 1/1 column families, dataSize=78 B heapSize=480 B
2024-12-04T15:44:49,891 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 4cea9c38548e4541884588dbfe1d8f02
2024-12-04T15:44:49,929 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/default/testBlockEvictionOnRegionMove1733327089160/2d7a4349fa18473dcc1fbca94949bd61/.tmp/family/4cea9c38548e4541884588dbfe1d8f02 is 43, key is row1/family:qf1/1/Put/seqid=0
2024-12-04T15:44:49,934 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/default/testBlockEvictionOnRegionMove1733327089160/2d7a4349fa18473dcc1fbca94949bd61/.tmp/family/4cea9c38548e4541884588dbfe1d8f02
2024-12-04T15:44:49,968 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/default/testBlockEvictionOnRegionMove1733327089160/2d7a4349fa18473dcc1fbca94949bd61/.tmp/family/4cea9c38548e4541884588dbfe1d8f02 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/default/testBlockEvictionOnRegionMove1733327089160/2d7a4349fa18473dcc1fbca94949bd61/family/4cea9c38548e4541884588dbfe1d8f02
2024-12-04T15:44:49,970 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/default/testBlockEvictionOnRegionMove1733327089160/2d7a4349fa18473dcc1fbca94949bd61/family/4cea9c38548e4541884588dbfe1d8f02, delay=992 ms
2024-12-04T15:44:49,971 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/default/testBlockEvictionOnRegionMove1733327089160/2d7a4349fa18473dcc1fbca94949bd61/family/4cea9c38548e4541884588dbfe1d8f02, entries=2, sequenceid=6, filesize=4.9 K
2024-12-04T15:44:49,977 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 2d7a4349fa18473dcc1fbca94949bd61 in 113ms, sequenceid=6, compaction requested=false
2024-12-04T15:44:49,978 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1733327089160'
2024-12-04T15:44:49,979 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 2d7a4349fa18473dcc1fbca94949bd61:

2024-12-04T15:44:50,964 DEBUG [hfile-prefetch-1733327089970 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 4cea9c38548e4541884588dbfe1d8f02_0. 
2024-12-04T15:44:50,968 DEBUG [hfile-prefetch-1733327089970 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 4cea9c38548e4541884588dbfe1d8f02_117. 
2024-12-04T15:44:50,969 DEBUG [hfile-prefetch-1733327089970 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/default/testBlockEvictionOnRegionMove1733327089160/2d7a4349fa18473dcc1fbca94949bd61/family/4cea9c38548e4541884588dbfe1d8f02, with total blocks 2, and data blocks 1
2024-12-04T15:44:50,969 DEBUG [hfile-prefetch-1733327089970 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map
2024-12-04T15:44:50,969 DEBUG [hfile-prefetch-1733327089970 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/default/testBlockEvictionOnRegionMove1733327089160/2d7a4349fa18473dcc1fbca94949bd61/family/4cea9c38548e4541884588dbfe1d8f02, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue.
2024-12-04T15:44:50,970 DEBUG [hfile-prefetch-1733327089970 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/default/testBlockEvictionOnRegionMove1733327089160/2d7a4349fa18473dcc1fbca94949bd61/family/4cea9c38548e4541884588dbfe1d8f02. Will sleep 100ms and try the verification again.
2024-12-04T15:44:50,984 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk
2024-12-04T15:44:50,984 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file
2024-12-04T15:44:50,984 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format.
2024-12-04T15:44:50,984 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000
2024-12-04T15:44:50,990 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43739 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=2d7a4349fa18473dcc1fbca94949bd61, source=a21b6491b371,40047,1733327087136, destination=a21b6491b371,46561,1733327086997, warming up region on a21b6491b371,46561,1733327086997
2024-12-04T15:44:50,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43739 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-04T15:44:50,993 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43739 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=2d7a4349fa18473dcc1fbca94949bd61, source=a21b6491b371,40047,1733327087136, destination=a21b6491b371,46561,1733327086997, running balancer
2024-12-04T15:44:50,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43739 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733327089160, region=2d7a4349fa18473dcc1fbca94949bd61, REOPEN/MOVE
2024-12-04T15:44:50,994 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42855, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-04T15:44:50,995 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733327089160, region=2d7a4349fa18473dcc1fbca94949bd61, REOPEN/MOVE
2024-12-04T15:44:50,996 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46561 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1733327089160,,1733327089182.2d7a4349fa18473dcc1fbca94949bd61.
2024-12-04T15:44:50,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46561 {}] regionserver.HRegion(7855): Warmup {ENCODED => 2d7a4349fa18473dcc1fbca94949bd61, NAME => 'testBlockEvictionOnRegionMove1733327089160,,1733327089182.2d7a4349fa18473dcc1fbca94949bd61.', STARTKEY => '', ENDKEY => ''}
2024-12-04T15:44:50,997 INFO  [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=2d7a4349fa18473dcc1fbca94949bd61, regionState=CLOSING, regionLocation=a21b6491b371,40047,1733327087136
2024-12-04T15:44:50,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46561 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733327089160,,1733327089182.2d7a4349fa18473dcc1fbca94949bd61.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-04T15:44:50,998 INFO  [StoreOpener-2d7a4349fa18473dcc1fbca94949bd61-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 2d7a4349fa18473dcc1fbca94949bd61 
2024-12-04T15:44:50,999 INFO  [StoreOpener-2d7a4349fa18473dcc1fbca94949bd61-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d7a4349fa18473dcc1fbca94949bd61 columnFamilyName family
2024-12-04T15:44:50,999 DEBUG [StoreOpener-2d7a4349fa18473dcc1fbca94949bd61-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-04T15:44:51,000 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733327089160, region=2d7a4349fa18473dcc1fbca94949bd61, REOPEN/MOVE because future has completed
2024-12-04T15:44:51,001 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true
2024-12-04T15:44:51,001 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2d7a4349fa18473dcc1fbca94949bd61, server=a21b6491b371,40047,1733327087136}]
2024-12-04T15:44:51,003 DEBUG [StoreOpener-2d7a4349fa18473dcc1fbca94949bd61-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/default/testBlockEvictionOnRegionMove1733327089160/2d7a4349fa18473dcc1fbca94949bd61/family/4cea9c38548e4541884588dbfe1d8f02
2024-12-04T15:44:51,004 INFO  [StoreOpener-2d7a4349fa18473dcc1fbca94949bd61-1 {}] regionserver.HStore(327): Store=2d7a4349fa18473dcc1fbca94949bd61/family,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-04T15:44:51,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46561 {}] regionserver.HRegion(1722): Closing 2d7a4349fa18473dcc1fbca94949bd61, disabling compactions & flushes
2024-12-04T15:44:51,004 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46561 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733327089160,,1733327089182.2d7a4349fa18473dcc1fbca94949bd61.
2024-12-04T15:44:51,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46561 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733327089160,,1733327089182.2d7a4349fa18473dcc1fbca94949bd61.
2024-12-04T15:44:51,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46561 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733327089160,,1733327089182.2d7a4349fa18473dcc1fbca94949bd61. after waiting 0 ms
2024-12-04T15:44:51,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46561 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733327089160,,1733327089182.2d7a4349fa18473dcc1fbca94949bd61.
2024-12-04T15:44:51,004 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0
2024-12-04T15:44:51,004 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733327090984
2024-12-04T15:44:51,005 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming
2024-12-04T15:44:51,008 DEBUG [StoreFileCloser-2d7a4349fa18473dcc1fbca94949bd61-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/default/testBlockEvictionOnRegionMove1733327089160/2d7a4349fa18473dcc1fbca94949bd61/family/4cea9c38548e4541884588dbfe1d8f02
2024-12-04T15:44:51,008 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46561 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733327089160,,1733327089182.2d7a4349fa18473dcc1fbca94949bd61.
2024-12-04T15:44:51,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46561 {}] regionserver.HRegion(1676): Region close journal for 2d7a4349fa18473dcc1fbca94949bd61:
Waiting for close lock at 1733327091004Disabling compacts and flushes for region at 1733327091004Disabling writes for close at 1733327091004Writing region close event to WAL at 1733327091008 (+4 ms)Closed at 1733327091008
2024-12-04T15:44:51,010 WARN  [hfile-prefetch-1733327089970 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/default/testBlockEvictionOnRegionMove1733327089160/2d7a4349fa18473dcc1fbca94949bd61/family/4cea9c38548e4541884588dbfe1d8f02, offset=158, end=158
java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted
	at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?]
	at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?]
	at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.lang.InterruptedException: sleep interrupted
	at java.lang.Thread.sleep(Native Method) ~[?:?]
	at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?]
	... 9 more
2024-12-04T15:44:51,011 DEBUG [hfile-prefetch-1733327089970 {}] hfile.PrefetchExecutor(121): Prefetch completed for 4cea9c38548e4541884588dbfe1d8f02
2024-12-04T15:44:51,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43739 {}] procedure.ProcedureSyncWait(219): waitFor pid=7
2024-12-04T15:44:51,159 INFO  [RS_CLOSE_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 2d7a4349fa18473dcc1fbca94949bd61
2024-12-04T15:44:51,159 DEBUG [RS_CLOSE_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true
2024-12-04T15:44:51,160 DEBUG [RS_CLOSE_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 2d7a4349fa18473dcc1fbca94949bd61, disabling compactions & flushes
2024-12-04T15:44:51,160 INFO  [RS_CLOSE_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733327089160,,1733327089182.2d7a4349fa18473dcc1fbca94949bd61.
2024-12-04T15:44:51,160 DEBUG [RS_CLOSE_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733327089160,,1733327089182.2d7a4349fa18473dcc1fbca94949bd61.
2024-12-04T15:44:51,160 DEBUG [RS_CLOSE_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733327089160,,1733327089182.2d7a4349fa18473dcc1fbca94949bd61. after waiting 0 ms
2024-12-04T15:44:51,160 DEBUG [RS_CLOSE_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733327089160,,1733327089182.2d7a4349fa18473dcc1fbca94949bd61.
2024-12-04T15:44:51,162 DEBUG [StoreFileCloser-2d7a4349fa18473dcc1fbca94949bd61-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file 4cea9c38548e4541884588dbfe1d8f02, starting offset: 0, end offset: 9223372036854775807
2024-12-04T15:44:51,164 DEBUG [StoreFileCloser-2d7a4349fa18473dcc1fbca94949bd61-family-1 {}] bucket.BucketCache(856): removed key 4cea9c38548e4541884588dbfe1d8f02_0 from back map with offset lock 2094080 in the evict process
2024-12-04T15:44:51,169 DEBUG [RS_CLOSE_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/default/testBlockEvictionOnRegionMove1733327089160/2d7a4349fa18473dcc1fbca94949bd61/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-12-04T15:44:51,170 INFO  [RS_CLOSE_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733327089160,,1733327089182.2d7a4349fa18473dcc1fbca94949bd61.
2024-12-04T15:44:51,170 DEBUG [RS_CLOSE_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 2d7a4349fa18473dcc1fbca94949bd61:
Waiting for close lock at 1733327091160Running coprocessor pre-close hooks at 1733327091160Disabling compacts and flushes for region at 1733327091160Disabling writes for close at 1733327091160Writing region close event to WAL at 1733327091164 (+4 ms)Running coprocessor post-close hooks at 1733327091169 (+5 ms)Closed at 1733327091170 (+1 ms)
2024-12-04T15:44:51,170 INFO  [RS_CLOSE_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 2d7a4349fa18473dcc1fbca94949bd61 move to a21b6491b371,46561,1733327086997 record at close sequenceid=6
2024-12-04T15:44:51,174 INFO  [RS_CLOSE_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 2d7a4349fa18473dcc1fbca94949bd61
2024-12-04T15:44:51,174 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=2d7a4349fa18473dcc1fbca94949bd61, regionState=CLOSED
2024-12-04T15:44:51,176 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2d7a4349fa18473dcc1fbca94949bd61, server=a21b6491b371,40047,1733327087136 because future has completed
2024-12-04T15:44:51,180 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7
2024-12-04T15:44:51,180 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 2d7a4349fa18473dcc1fbca94949bd61, server=a21b6491b371,40047,1733327087136 in 177 msec
2024-12-04T15:44:51,180 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733327089160, region=2d7a4349fa18473dcc1fbca94949bd61, REOPEN/MOVE; state=CLOSED, location=a21b6491b371,46561,1733327086997; forceNewPlan=false, retain=false
2024-12-04T15:44:51,205 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk
2024-12-04T15:44:51,205 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file
2024-12-04T15:44:51,205 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format.
2024-12-04T15:44:51,205 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000
2024-12-04T15:44:51,208 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0
2024-12-04T15:44:51,209 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733327091205
2024-12-04T15:44:51,209 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming
2024-12-04T15:44:51,331 INFO  [a21b6491b371:43739 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 
2024-12-04T15:44:51,331 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=2d7a4349fa18473dcc1fbca94949bd61, regionState=OPENING, regionLocation=a21b6491b371,46561,1733327086997
2024-12-04T15:44:51,335 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733327089160, region=2d7a4349fa18473dcc1fbca94949bd61, REOPEN/MOVE because future has completed
2024-12-04T15:44:51,335 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2d7a4349fa18473dcc1fbca94949bd61, server=a21b6491b371,46561,1733327086997}]
2024-12-04T15:44:51,490 INFO  [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1733327089160,,1733327089182.2d7a4349fa18473dcc1fbca94949bd61.
2024-12-04T15:44:51,491 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 2d7a4349fa18473dcc1fbca94949bd61, NAME => 'testBlockEvictionOnRegionMove1733327089160,,1733327089182.2d7a4349fa18473dcc1fbca94949bd61.', STARTKEY => '', ENDKEY => ''}
2024-12-04T15:44:51,491 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1733327089160 2d7a4349fa18473dcc1fbca94949bd61
2024-12-04T15:44:51,491 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733327089160,,1733327089182.2d7a4349fa18473dcc1fbca94949bd61.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-04T15:44:51,491 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 2d7a4349fa18473dcc1fbca94949bd61
2024-12-04T15:44:51,491 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 2d7a4349fa18473dcc1fbca94949bd61
2024-12-04T15:44:51,492 INFO  [StoreOpener-2d7a4349fa18473dcc1fbca94949bd61-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 2d7a4349fa18473dcc1fbca94949bd61 
2024-12-04T15:44:51,493 INFO  [StoreOpener-2d7a4349fa18473dcc1fbca94949bd61-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d7a4349fa18473dcc1fbca94949bd61 columnFamilyName family
2024-12-04T15:44:51,493 DEBUG [StoreOpener-2d7a4349fa18473dcc1fbca94949bd61-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-04T15:44:51,495 DEBUG [StoreFileOpener-2d7a4349fa18473dcc1fbca94949bd61-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/default/testBlockEvictionOnRegionMove1733327089160/2d7a4349fa18473dcc1fbca94949bd61/family/4cea9c38548e4541884588dbfe1d8f02, delay=919 ms
2024-12-04T15:44:51,496 DEBUG [StoreOpener-2d7a4349fa18473dcc1fbca94949bd61-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/default/testBlockEvictionOnRegionMove1733327089160/2d7a4349fa18473dcc1fbca94949bd61/family/4cea9c38548e4541884588dbfe1d8f02
2024-12-04T15:44:51,496 INFO  [StoreOpener-2d7a4349fa18473dcc1fbca94949bd61-1 {}] regionserver.HStore(327): Store=2d7a4349fa18473dcc1fbca94949bd61/family,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-04T15:44:51,497 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 2d7a4349fa18473dcc1fbca94949bd61
2024-12-04T15:44:51,498 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/default/testBlockEvictionOnRegionMove1733327089160/2d7a4349fa18473dcc1fbca94949bd61
2024-12-04T15:44:51,498 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/default/testBlockEvictionOnRegionMove1733327089160/2d7a4349fa18473dcc1fbca94949bd61
2024-12-04T15:44:51,498 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 2d7a4349fa18473dcc1fbca94949bd61
2024-12-04T15:44:51,498 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 2d7a4349fa18473dcc1fbca94949bd61
2024-12-04T15:44:51,503 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 2d7a4349fa18473dcc1fbca94949bd61
2024-12-04T15:44:51,503 INFO  [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 2d7a4349fa18473dcc1fbca94949bd61; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68277437, jitterRate=0.017413094639778137}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-04T15:44:51,503 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2d7a4349fa18473dcc1fbca94949bd61
2024-12-04T15:44:51,504 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 2d7a4349fa18473dcc1fbca94949bd61:
Running coprocessor pre-open hook at 1733327091491Writing region info on filesystem at 1733327091491Initializing all the Stores at 1733327091492 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733327091492Cleaning up temporary data from old regions at 1733327091498 (+6 ms)Running coprocessor post-open hooks at 1733327091503 (+5 ms)Region opened successfully at 1733327091504 (+1 ms)
2024-12-04T15:44:51,505 INFO  [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1733327089160,,1733327089182.2d7a4349fa18473dcc1fbca94949bd61., pid=9, masterSystemTime=1733327091488
2024-12-04T15:44:51,507 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1733327089160,,1733327089182.2d7a4349fa18473dcc1fbca94949bd61.
2024-12-04T15:44:51,507 INFO  [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1733327089160,,1733327089182.2d7a4349fa18473dcc1fbca94949bd61.
2024-12-04T15:44:51,508 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=2d7a4349fa18473dcc1fbca94949bd61, regionState=OPEN, openSeqNum=10, regionLocation=a21b6491b371,46561,1733327086997
2024-12-04T15:44:51,511 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2d7a4349fa18473dcc1fbca94949bd61, server=a21b6491b371,46561,1733327086997 because future has completed
2024-12-04T15:44:51,514 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7
2024-12-04T15:44:51,514 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 2d7a4349fa18473dcc1fbca94949bd61, server=a21b6491b371,46561,1733327086997 in 178 msec
2024-12-04T15:44:51,516 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733327089160, region=2d7a4349fa18473dcc1fbca94949bd61, REOPEN/MOVE in 521 msec
2024-12-04T15:44:51,547 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-04T15:44:51,547 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-04T15:44:51,548 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-04T15:44:51,552 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-04T15:44:51,553 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-04T15:44:51,553 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-04T15:44:51,553 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-04T15:44:51,553 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=520236456, stopped=false
2024-12-04T15:44:51,553 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=a21b6491b371,43739,1733327086216
2024-12-04T15:44:51,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43739-0x1017e264bf50000, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-04T15:44:51,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40047-0x1017e264bf50002, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-04T15:44:51,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46561-0x1017e264bf50001, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-04T15:44:51,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43739-0x1017e264bf50000, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-04T15:44:51,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40047-0x1017e264bf50002, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-04T15:44:51,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46561-0x1017e264bf50001, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-04T15:44:51,556 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-04T15:44:51,557 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40047-0x1017e264bf50002, quorum=127.0.0.1:64258, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-04T15:44:51,557 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43739-0x1017e264bf50000, quorum=127.0.0.1:64258, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-04T15:44:51,557 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46561-0x1017e264bf50001, quorum=127.0.0.1:64258, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-04T15:44:51,557 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-04T15:44:51,557 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-04T15:44:51,557 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-04T15:44:51,558 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a21b6491b371,46561,1733327086997' *****
2024-12-04T15:44:51,558 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-04T15:44:51,558 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a21b6491b371,40047,1733327087136' *****
2024-12-04T15:44:51,558 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-04T15:44:51,558 INFO  [RS:1;a21b6491b371:40047 {}] bucket.BucketCache(1738): Disabling cache
2024-12-04T15:44:51,558 INFO  [RS:0;a21b6491b371:46561 {}] bucket.BucketCache(1738): Disabling cache
2024-12-04T15:44:51,558 INFO  [RS:1;a21b6491b371:40047 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/bucket.persistence
2024-12-04T15:44:51,558 INFO  [RS:0;a21b6491b371:46561 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/bucket.persistence
2024-12-04T15:44:51,558 INFO  [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false
2024-12-04T15:44:51,558 INFO  [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false
2024-12-04T15:44:51,558 INFO  [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false
2024-12-04T15:44:51,558 INFO  [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false
2024-12-04T15:44:51,559 INFO  [RS:1;a21b6491b371:40047 {}] bucket.BucketCache(1768): Shutting down cache persister thread.
2024-12-04T15:44:51,559 INFO  [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false
2024-12-04T15:44:51,559 INFO  [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false
2024-12-04T15:44:51,559 INFO  [RS:0;a21b6491b371:46561 {}] bucket.BucketCache(1768): Shutting down cache persister thread.
2024-12-04T15:44:51,559 WARN  [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread.
java.lang.InterruptedException: sleep interrupted
	at java.lang.Thread.sleep(Native Method) ~[?:?]
	at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?]
2024-12-04T15:44:51,559 WARN  [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread.
java.lang.InterruptedException: sleep interrupted
	at java.lang.Thread.sleep(Native Method) ~[?:?]
	at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?]
2024-12-04T15:44:51,561 INFO  [regionserver/a21b6491b371:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-04T15:44:51,561 INFO  [regionserver/a21b6491b371:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-04T15:44:51,562 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-04T15:44:51,562 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-04T15:44:51,569 DEBUG [RS:1;a21b6491b371:40047 {}] bucket.BucketCache(1487): Thread RS:1;a21b6491b371:40047 started persisting bucket cache to file
2024-12-04T15:44:51,569 DEBUG [RS:0;a21b6491b371:46561 {}] bucket.BucketCache(1487): Thread RS:0;a21b6491b371:46561 started persisting bucket cache to file
2024-12-04T15:44:51,569 DEBUG [RS:1;a21b6491b371:40047 {}] bucket.BucketCache(1494): Persist in new chunked persistence format.
2024-12-04T15:44:51,569 DEBUG [RS:1;a21b6491b371:40047 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000
2024-12-04T15:44:51,569 DEBUG [RS:0;a21b6491b371:46561 {}] bucket.BucketCache(1494): Persist in new chunked persistence format.
2024-12-04T15:44:51,569 DEBUG [RS:0;a21b6491b371:46561 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000
2024-12-04T15:44:51,573 DEBUG [RS:0;a21b6491b371:46561 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0
2024-12-04T15:44:51,573 DEBUG [RS:0;a21b6491b371:46561 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733327091569
2024-12-04T15:44:51,573 DEBUG [RS:1;a21b6491b371:40047 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0
2024-12-04T15:44:51,573 DEBUG [RS:0;a21b6491b371:46561 {}] bucket.BucketCache(1509): Thread RS:0;a21b6491b371:46561 finished persisting bucket cache to file, renaming
2024-12-04T15:44:51,573 DEBUG [RS:1;a21b6491b371:40047 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733327091569
2024-12-04T15:44:51,573 DEBUG [RS:1;a21b6491b371:40047 {}] bucket.BucketCache(1509): Thread RS:1;a21b6491b371:40047 finished persisting bucket cache to file, renaming
2024-12-04T15:44:51,573 WARN  [RS:1;a21b6491b371:40047 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again.
2024-12-04T15:44:51,573 INFO  [RS:1;a21b6491b371:40047 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-04T15:44:51,573 INFO  [RS:0;a21b6491b371:46561 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-04T15:44:51,573 INFO  [RS:1;a21b6491b371:40047 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-04T15:44:51,573 INFO  [RS:0;a21b6491b371:46561 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-04T15:44:51,574 INFO  [RS:1;a21b6491b371:40047 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-04T15:44:51,574 INFO  [RS:0;a21b6491b371:46561 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-04T15:44:51,574 INFO  [RS:1;a21b6491b371:40047 {}] regionserver.HRegionServer(959): stopping server a21b6491b371,40047,1733327087136
2024-12-04T15:44:51,574 INFO  [RS:1;a21b6491b371:40047 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-04T15:44:51,574 INFO  [RS:0;a21b6491b371:46561 {}] regionserver.HRegionServer(3091): Received CLOSE for 2d7a4349fa18473dcc1fbca94949bd61
2024-12-04T15:44:51,574 INFO  [RS:1;a21b6491b371:40047 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;a21b6491b371:40047.
2024-12-04T15:44:51,574 DEBUG [RS:1;a21b6491b371:40047 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-04T15:44:51,574 DEBUG [RS:1;a21b6491b371:40047 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-04T15:44:51,574 INFO  [RS:1;a21b6491b371:40047 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-04T15:44:51,574 INFO  [RS:1;a21b6491b371:40047 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-04T15:44:51,574 INFO  [RS:1;a21b6491b371:40047 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-04T15:44:51,574 INFO  [RS:1;a21b6491b371:40047 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-04T15:44:51,574 INFO  [RS:0;a21b6491b371:46561 {}] regionserver.HRegionServer(959): stopping server a21b6491b371,46561,1733327086997
2024-12-04T15:44:51,574 INFO  [RS:0;a21b6491b371:46561 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-04T15:44:51,574 INFO  [RS:0;a21b6491b371:46561 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;a21b6491b371:46561.
2024-12-04T15:44:51,574 DEBUG [RS:0;a21b6491b371:46561 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-04T15:44:51,575 DEBUG [RS:0;a21b6491b371:46561 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-04T15:44:51,575 INFO  [RS:1;a21b6491b371:40047 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-12-04T15:44:51,575 INFO  [RS:0;a21b6491b371:46561 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-12-04T15:44:51,575 DEBUG [RS:0;a21b6491b371:46561 {}] regionserver.HRegionServer(1325): Online Regions={2d7a4349fa18473dcc1fbca94949bd61=testBlockEvictionOnRegionMove1733327089160,,1733327089182.2d7a4349fa18473dcc1fbca94949bd61.}
2024-12-04T15:44:51,575 DEBUG [RS:1;a21b6491b371:40047 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-12-04T15:44:51,575 INFO  [RS_CLOSE_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1733327089160,,1733327089182.2d7a4349fa18473dcc1fbca94949bd61. during a graceful stop, and cache persistence is on, so setting evict on close to false. 
2024-12-04T15:44:51,575 INFO  [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 
2024-12-04T15:44:51,575 DEBUG [RS_CLOSE_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2d7a4349fa18473dcc1fbca94949bd61, disabling compactions & flushes
2024-12-04T15:44:51,575 INFO  [RS_CLOSE_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733327089160,,1733327089182.2d7a4349fa18473dcc1fbca94949bd61.
2024-12-04T15:44:51,575 DEBUG [RS:1;a21b6491b371:40047 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-12-04T15:44:51,575 DEBUG [RS:0;a21b6491b371:46561 {}] regionserver.HRegionServer(1351): Waiting on 2d7a4349fa18473dcc1fbca94949bd61
2024-12-04T15:44:51,575 DEBUG [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-04T15:44:51,575 DEBUG [RS_CLOSE_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733327089160,,1733327089182.2d7a4349fa18473dcc1fbca94949bd61.
2024-12-04T15:44:51,575 DEBUG [RS_CLOSE_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733327089160,,1733327089182.2d7a4349fa18473dcc1fbca94949bd61. after waiting 0 ms
2024-12-04T15:44:51,575 INFO  [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-04T15:44:51,575 DEBUG [RS_CLOSE_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733327089160,,1733327089182.2d7a4349fa18473dcc1fbca94949bd61.
2024-12-04T15:44:51,575 DEBUG [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-04T15:44:51,575 DEBUG [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-04T15:44:51,576 DEBUG [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-04T15:44:51,576 INFO  [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB
2024-12-04T15:44:51,576 DEBUG [StoreFileCloser-2d7a4349fa18473dcc1fbca94949bd61-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/default/testBlockEvictionOnRegionMove1733327089160/2d7a4349fa18473dcc1fbca94949bd61/family/4cea9c38548e4541884588dbfe1d8f02
2024-12-04T15:44:51,581 DEBUG [RS_CLOSE_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/default/testBlockEvictionOnRegionMove1733327089160/2d7a4349fa18473dcc1fbca94949bd61/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9
2024-12-04T15:44:51,581 INFO  [RS_CLOSE_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733327089160,,1733327089182.2d7a4349fa18473dcc1fbca94949bd61.
2024-12-04T15:44:51,582 DEBUG [RS_CLOSE_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2d7a4349fa18473dcc1fbca94949bd61:
Waiting for close lock at 1733327091574Running coprocessor pre-close hooks at 1733327091575 (+1 ms)Disabling compacts and flushes for region at 1733327091575Disabling writes for close at 1733327091575Writing region close event to WAL at 1733327091576 (+1 ms)Running coprocessor post-close hooks at 1733327091581 (+5 ms)Closed at 1733327091581
2024-12-04T15:44:51,582 DEBUG [RS_CLOSE_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1733327089160,,1733327089182.2d7a4349fa18473dcc1fbca94949bd61.
2024-12-04T15:44:51,587 DEBUG [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d291b6bc6a7b486fbb11c411278d78f0
2024-12-04T15:44:51,605 DEBUG [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/.tmp/info/d291b6bc6a7b486fbb11c411278d78f0 is 205, key is testBlockEvictionOnRegionMove1733327089160,,1733327089182.2d7a4349fa18473dcc1fbca94949bd61./info:regioninfo/1733327091508/Put/seqid=0
2024-12-04T15:44:51,606 INFO  [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/.tmp/info/d291b6bc6a7b486fbb11c411278d78f0
2024-12-04T15:44:51,616 DEBUG [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a90d0387ed7b44cdb62564c5fcb053a9
2024-12-04T15:44:51,633 DEBUG [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/.tmp/ns/a90d0387ed7b44cdb62564c5fcb053a9 is 43, key is default/ns:d/1733327088978/Put/seqid=0
2024-12-04T15:44:51,634 INFO  [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/.tmp/ns/a90d0387ed7b44cdb62564c5fcb053a9
2024-12-04T15:44:51,645 DEBUG [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 57c7555af3f3425dacca1e40ac98a599
2024-12-04T15:44:51,665 DEBUG [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/.tmp/table/57c7555af3f3425dacca1e40ac98a599 is 78, key is testBlockEvictionOnRegionMove1733327089160/table:state/1733327089596/Put/seqid=0
2024-12-04T15:44:51,666 INFO  [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/.tmp/table/57c7555af3f3425dacca1e40ac98a599
2024-12-04T15:44:51,670 DEBUG [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/.tmp/info/d291b6bc6a7b486fbb11c411278d78f0 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/info/d291b6bc6a7b486fbb11c411278d78f0
2024-12-04T15:44:51,672 DEBUG [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/info/d291b6bc6a7b486fbb11c411278d78f0, delay=918 ms
2024-12-04T15:44:51,673 INFO  [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/info/d291b6bc6a7b486fbb11c411278d78f0, entries=15, sequenceid=15, filesize=7.6 K
2024-12-04T15:44:51,674 DEBUG [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/.tmp/ns/a90d0387ed7b44cdb62564c5fcb053a9 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/ns/a90d0387ed7b44cdb62564c5fcb053a9
2024-12-04T15:44:51,675 DEBUG [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/ns/a90d0387ed7b44cdb62564c5fcb053a9, delay=931 ms
2024-12-04T15:44:51,676 INFO  [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/ns/a90d0387ed7b44cdb62564c5fcb053a9, entries=2, sequenceid=15, filesize=5.0 K
2024-12-04T15:44:51,677 DEBUG [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/.tmp/table/57c7555af3f3425dacca1e40ac98a599 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/table/57c7555af3f3425dacca1e40ac98a599
2024-12-04T15:44:51,678 DEBUG [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/table/57c7555af3f3425dacca1e40ac98a599, delay=954 ms
2024-12-04T15:44:51,679 INFO  [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/table/57c7555af3f3425dacca1e40ac98a599, entries=2, sequenceid=15, filesize=5.3 K
2024-12-04T15:44:51,680 INFO  [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 104ms, sequenceid=15, compaction requested=false
2024-12-04T15:44:51,680 DEBUG [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-12-04T15:44:51,681 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/info/d291b6bc6a7b486fbb11c411278d78f0
2024-12-04T15:44:51,682 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/ns/a90d0387ed7b44cdb62564c5fcb053a9
2024-12-04T15:44:51,683 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/table/57c7555af3f3425dacca1e40ac98a599
2024-12-04T15:44:51,687 DEBUG [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1
2024-12-04T15:44:51,688 DEBUG [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-04T15:44:51,688 INFO  [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-04T15:44:51,688 DEBUG [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733327091575Running coprocessor pre-close hooks at 1733327091575Disabling compacts and flushes for region at 1733327091575Disabling writes for close at 1733327091575Obtaining lock to block concurrent updates at 1733327091576 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1733327091576Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1733327091576Flushing stores of hbase:meta,,1.1588230740 at 1733327091577 (+1 ms)Flushing 1588230740/info: creating writer at 1733327091577Flushing 1588230740/info: appending metadata at 1733327091604 (+27 ms)Flushing 1588230740/info: closing flushed file at 1733327091604Flushing 1588230740/ns: creating writer at 1733327091609 (+5 ms)Flushing 1588230740/ns: appending metadata at 1733327091632 (+23 ms)Flushing 1588230740/ns: closing flushed file at 1733327091632Flushing 1588230740/table: creating writer at 1733327091638 (+6 ms)Flushing 1588230740/table: appending metadata at 1733327091664 (+26 ms)Flushing 1588230740/table: closing flushed file at 1733327091664Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37922bff: reopening flushed file at 1733327091669 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7310970f: reopening flushed file at 1733327091673 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a98b37c: reopening flushed file at 1733327091676 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 104ms, sequenceid=15, compaction requested=false at 1733327091680 (+4 ms)Writing region close event to WAL at 1733327091683 (+3 ms)Running coprocessor post-close hooks at 1733327091687 (+4 ms)Closed at 1733327091688 (+1 ms)
2024-12-04T15:44:51,689 DEBUG [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-04T15:44:51,776 INFO  [RS:0;a21b6491b371:46561 {}] regionserver.HRegionServer(976): stopping server a21b6491b371,46561,1733327086997; all regions closed.
2024-12-04T15:44:51,776 INFO  [RS:1;a21b6491b371:40047 {}] regionserver.HRegionServer(976): stopping server a21b6491b371,40047,1733327087136; all regions closed.
2024-12-04T15:44:51,781 DEBUG [RS:1;a21b6491b371:40047 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/oldWALs
2024-12-04T15:44:51,781 DEBUG [RS:0;a21b6491b371:46561 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/oldWALs
2024-12-04T15:44:51,781 INFO  [RS:1;a21b6491b371:40047 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a21b6491b371%2C40047%2C1733327087136.meta:.meta(num 1733327088841)
2024-12-04T15:44:51,781 INFO  [RS:0;a21b6491b371:46561 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a21b6491b371%2C46561%2C1733327086997:(num 1733327088519)
2024-12-04T15:44:51,781 DEBUG [RS:0;a21b6491b371:46561 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-04T15:44:51,781 INFO  [RS:0;a21b6491b371:46561 {}] regionserver.LeaseManager(133): Closed leases
2024-12-04T15:44:51,781 INFO  [RS:0;a21b6491b371:46561 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-04T15:44:51,782 INFO  [RS:0;a21b6491b371:46561 {}] hbase.ChoreService(370): Chore service for: regionserver/a21b6491b371:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-12-04T15:44:51,782 INFO  [RS:0;a21b6491b371:46561 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-04T15:44:51,782 INFO  [regionserver/a21b6491b371:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-04T15:44:51,782 INFO  [RS:0;a21b6491b371:46561 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-04T15:44:51,782 INFO  [RS:0;a21b6491b371:46561 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-04T15:44:51,782 INFO  [RS:0;a21b6491b371:46561 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-04T15:44:51,783 INFO  [RS:0;a21b6491b371:46561 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46561
2024-12-04T15:44:51,783 DEBUG [RS:1;a21b6491b371:40047 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/oldWALs
2024-12-04T15:44:51,783 INFO  [RS:1;a21b6491b371:40047 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a21b6491b371%2C40047%2C1733327087136:(num 1733327088519)
2024-12-04T15:44:51,783 DEBUG [RS:1;a21b6491b371:40047 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-04T15:44:51,783 INFO  [RS:1;a21b6491b371:40047 {}] regionserver.LeaseManager(133): Closed leases
2024-12-04T15:44:51,783 INFO  [RS:1;a21b6491b371:40047 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-04T15:44:51,784 INFO  [RS:1;a21b6491b371:40047 {}] hbase.ChoreService(370): Chore service for: regionserver/a21b6491b371:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-12-04T15:44:51,784 INFO  [RS:1;a21b6491b371:40047 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-04T15:44:51,784 INFO  [regionserver/a21b6491b371:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-04T15:44:51,784 INFO  [RS:1;a21b6491b371:40047 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40047
2024-12-04T15:44:51,789 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46561-0x1017e264bf50001, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a21b6491b371,46561,1733327086997
2024-12-04T15:44:51,789 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43739-0x1017e264bf50000, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-04T15:44:51,789 INFO  [RS:0;a21b6491b371:46561 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-04T15:44:51,790 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40047-0x1017e264bf50002, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a21b6491b371,40047,1733327087136
2024-12-04T15:44:51,790 INFO  [RS:1;a21b6491b371:40047 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-04T15:44:51,792 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a21b6491b371,40047,1733327087136]
2024-12-04T15:44:51,793 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a21b6491b371,40047,1733327087136 already deleted, retry=false
2024-12-04T15:44:51,793 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a21b6491b371,40047,1733327087136 expired; onlineServers=1
2024-12-04T15:44:51,793 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a21b6491b371,46561,1733327086997]
2024-12-04T15:44:51,796 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a21b6491b371,46561,1733327086997 already deleted, retry=false
2024-12-04T15:44:51,796 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a21b6491b371,46561,1733327086997 expired; onlineServers=0
2024-12-04T15:44:51,796 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'a21b6491b371,43739,1733327086216' *****
2024-12-04T15:44:51,796 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-04T15:44:51,796 INFO  [M:0;a21b6491b371:43739 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-04T15:44:51,796 INFO  [M:0;a21b6491b371:43739 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-04T15:44:51,796 DEBUG [M:0;a21b6491b371:43739 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-04T15:44:51,797 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-04T15:44:51,797 DEBUG [M:0;a21b6491b371:43739 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-04T15:44:51,797 DEBUG [master/a21b6491b371:0:becomeActiveMaster-HFileCleaner.small.0-1733327088125 {}] cleaner.HFileCleaner(306): Exit Thread[master/a21b6491b371:0:becomeActiveMaster-HFileCleaner.small.0-1733327088125,5,FailOnTimeoutGroup]
2024-12-04T15:44:51,797 DEBUG [master/a21b6491b371:0:becomeActiveMaster-HFileCleaner.large.0-1733327088122 {}] cleaner.HFileCleaner(306): Exit Thread[master/a21b6491b371:0:becomeActiveMaster-HFileCleaner.large.0-1733327088122,5,FailOnTimeoutGroup]
2024-12-04T15:44:51,797 INFO  [M:0;a21b6491b371:43739 {}] hbase.ChoreService(370): Chore service for: master/a21b6491b371:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-04T15:44:51,797 INFO  [M:0;a21b6491b371:43739 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-04T15:44:51,797 DEBUG [M:0;a21b6491b371:43739 {}] master.HMaster(1795): Stopping service threads
2024-12-04T15:44:51,797 INFO  [M:0;a21b6491b371:43739 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-04T15:44:51,797 INFO  [M:0;a21b6491b371:43739 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-04T15:44:51,798 INFO  [M:0;a21b6491b371:43739 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-04T15:44:51,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43739-0x1017e264bf50000, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-04T15:44:51,798 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-04T15:44:51,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43739-0x1017e264bf50000, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-04T15:44:51,799 DEBUG [M:0;a21b6491b371:43739 {}] zookeeper.ZKUtil(347): master:43739-0x1017e264bf50000, quorum=127.0.0.1:64258, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-04T15:44:51,799 WARN  [M:0;a21b6491b371:43739 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-04T15:44:51,799 INFO  [M:0;a21b6491b371:43739 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/.lastflushedseqids
2024-12-04T15:44:51,808 INFO  [M:0;a21b6491b371:43739 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-04T15:44:51,808 INFO  [M:0;a21b6491b371:43739 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-04T15:44:51,809 DEBUG [M:0;a21b6491b371:43739 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-04T15:44:51,809 INFO  [M:0;a21b6491b371:43739 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-04T15:44:51,809 DEBUG [M:0;a21b6491b371:43739 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-04T15:44:51,809 DEBUG [M:0;a21b6491b371:43739 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-04T15:44:51,809 DEBUG [M:0;a21b6491b371:43739 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-04T15:44:51,809 INFO  [M:0;a21b6491b371:43739 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.69 KB heapSize=43.02 KB
2024-12-04T15:44:51,822 DEBUG [M:0;a21b6491b371:43739 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6f4050ab362e45809d7f84b5397b0739
2024-12-04T15:44:51,836 DEBUG [M:0;a21b6491b371:43739 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6f4050ab362e45809d7f84b5397b0739 is 82, key is hbase:meta,,1/info:regioninfo/1733327088906/Put/seqid=0
2024-12-04T15:44:51,837 INFO  [M:0;a21b6491b371:43739 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6f4050ab362e45809d7f84b5397b0739
2024-12-04T15:44:51,847 DEBUG [M:0;a21b6491b371:43739 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 0699350af2124f7b9fa8249c0798883e
2024-12-04T15:44:51,861 DEBUG [M:0;a21b6491b371:43739 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0699350af2124f7b9fa8249c0798883e is 779, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733327089602/Put/seqid=0
2024-12-04T15:44:51,862 INFO  [M:0;a21b6491b371:43739 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.07 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0699350af2124f7b9fa8249c0798883e
2024-12-04T15:44:51,871 DEBUG [M:0;a21b6491b371:43739 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8e4fab85ea6b496796d7446204801b46
2024-12-04T15:44:51,883 DEBUG [M:0;a21b6491b371:43739 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8e4fab85ea6b496796d7446204801b46 is 69, key is a21b6491b371,40047,1733327087136/rs:state/1733327088269/Put/seqid=0
2024-12-04T15:44:51,884 INFO  [M:0;a21b6491b371:43739 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8e4fab85ea6b496796d7446204801b46
2024-12-04T15:44:51,886 DEBUG [M:0;a21b6491b371:43739 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6f4050ab362e45809d7f84b5397b0739 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/6f4050ab362e45809d7f84b5397b0739
2024-12-04T15:44:51,887 INFO  [M:0;a21b6491b371:43739 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/6f4050ab362e45809d7f84b5397b0739, entries=8, sequenceid=84, filesize=5.5 K
2024-12-04T15:44:51,888 DEBUG [M:0;a21b6491b371:43739 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0699350af2124f7b9fa8249c0798883e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0699350af2124f7b9fa8249c0798883e
2024-12-04T15:44:51,889 INFO  [M:0;a21b6491b371:43739 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0699350af2124f7b9fa8249c0798883e, entries=9, sequenceid=84, filesize=6.5 K
2024-12-04T15:44:51,890 DEBUG [M:0;a21b6491b371:43739 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8e4fab85ea6b496796d7446204801b46 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8e4fab85ea6b496796d7446204801b46
2024-12-04T15:44:51,891 INFO  [M:0;a21b6491b371:43739 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8e4fab85ea6b496796d7446204801b46, entries=2, sequenceid=84, filesize=5.1 K
2024-12-04T15:44:51,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46561-0x1017e264bf50001, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-04T15:44:51,892 INFO  [M:0;a21b6491b371:43739 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 83ms, sequenceid=84, compaction requested=false
2024-12-04T15:44:51,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46561-0x1017e264bf50001, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-04T15:44:51,892 INFO  [RS:0;a21b6491b371:46561 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-04T15:44:51,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40047-0x1017e264bf50002, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-04T15:44:51,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40047-0x1017e264bf50002, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-04T15:44:51,892 INFO  [RS:1;a21b6491b371:40047 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-04T15:44:51,893 INFO  [RS:0;a21b6491b371:46561 {}] regionserver.HRegionServer(1031): Exiting; stopping=a21b6491b371,46561,1733327086997; zookeeper connection closed.
2024-12-04T15:44:51,893 INFO  [RS:1;a21b6491b371:40047 {}] regionserver.HRegionServer(1031): Exiting; stopping=a21b6491b371,40047,1733327087136; zookeeper connection closed.
2024-12-04T15:44:51,893 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@237ccb41 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@237ccb41
2024-12-04T15:44:51,893 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1464eba6 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1464eba6
2024-12-04T15:44:51,894 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete
2024-12-04T15:44:51,894 INFO  [M:0;a21b6491b371:43739 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-04T15:44:51,894 DEBUG [M:0;a21b6491b371:43739 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733327091809Disabling compacts and flushes for region at 1733327091809Disabling writes for close at 1733327091809Obtaining lock to block concurrent updates at 1733327091809Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733327091809Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35518, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1733327091810 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733327091810Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733327091810Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733327091835 (+25 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733327091835Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733327091839 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733327091860 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733327091860Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733327091863 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733327091883 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733327091883Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2cf02c01: reopening flushed file at 1733327091885 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@76ea1628: reopening flushed file at 1733327091887 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ac9ad73: reopening flushed file at 1733327091889 (+2 ms)Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 83ms, sequenceid=84, compaction requested=false at 1733327091892 (+3 ms)Writing region close event to WAL at 1733327091894 (+2 ms)Closed at 1733327091894
2024-12-04T15:44:51,897 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-04T15:44:51,897 INFO  [M:0;a21b6491b371:43739 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-04T15:44:51,897 INFO  [M:0;a21b6491b371:43739 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43739
2024-12-04T15:44:51,898 INFO  [M:0;a21b6491b371:43739 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-04T15:44:52,001 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43739-0x1017e264bf50000, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-04T15:44:52,001 INFO  [M:0;a21b6491b371:43739 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-04T15:44:52,001 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43739-0x1017e264bf50000, quorum=127.0.0.1:64258, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-04T15:44:52,006 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-04T15:44:52,028 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-04T15:44:52,029 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-04T15:44:52,038 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=49 (was 12)
Potentially hanging thread: master/a21b6491b371:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: hfile-prefetch-1733327089970
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: regionserver/a21b6491b371:0.procedureResultReporter
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75)

Potentially hanging thread: HBase-Metrics2-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Async-Client-Retry-Timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool.commonPool-worker-2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: hfile-prefetch-1733327091672
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-1-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Time-limited test.named-queue-events-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47)
	app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56)
	app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159)
	app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Timer for 'HBase' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: region-location-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Monitor thread for TaskMonitor
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: weak-ref-cleaner-strictcontextstorage
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: region-location-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool.commonPool-worker-3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: regionserver/a21b6491b371:0.procedureResultReporter
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75)

Potentially hanging thread: HMaster-EventLoopGroup-1-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: SnapshotHandlerChoreCleaner
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: master/a21b6491b371:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: hfile-prefetch-1733327091495
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: hfile-prefetch-1733327091675
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool.commonPool-worker-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: SessionTracker
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163)

Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RpcClient-timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-1-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=902 (was 964), ProcessCount=11 (was 11), AvailableMemoryMB=9910 (was 10256)
2024-12-04T15:44:52,044 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=49, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=902, ProcessCount=11, AvailableMemoryMB=9910
2024-12-04T15:44:52,044 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/hadoop-log-dir so I do NOT create it in target/test-data/de2c1fb8-a351-04c3-cfcb-b73746afa260
2024-12-04T15:44:52,044 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/hadoop-tmp-dir so I do NOT create it in target/test-data/de2c1fb8-a351-04c3-cfcb-b73746afa260
2024-12-04T15:44:52,045 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-04T15:44:52,048 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/de2c1fb8-a351-04c3-cfcb-b73746afa260/cluster_efeb1186-e87b-371e-a2d6-009744c0ef22, deleteOnExit=true
2024-12-04T15:44:52,050 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/de2c1fb8-a351-04c3-cfcb-b73746afa260/cluster_efeb1186-e87b-371e-a2d6-009744c0ef22/zookeeper_0, clientPort=58881, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/de2c1fb8-a351-04c3-cfcb-b73746afa260/cluster_efeb1186-e87b-371e-a2d6-009744c0ef22/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/de2c1fb8-a351-04c3-cfcb-b73746afa260/cluster_efeb1186-e87b-371e-a2d6-009744c0ef22/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-12-04T15:44:52,051 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58881
2024-12-04T15:44:52,064 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621 with version=8
2024-12-04T15:44:52,064 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/hbase-staging
2024-12-04T15:44:52,066 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/a21b6491b371:0 server-side Connection retries=45
2024-12-04T15:44:52,066 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-04T15:44:52,066 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-04T15:44:52,067 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-04T15:44:52,067 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-04T15:44:52,067 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-04T15:44:52,067 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-12-04T15:44:52,067 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-04T15:44:52,068 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44449
2024-12-04T15:44:52,069 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44449 connecting to ZooKeeper ensemble=127.0.0.1:58881
2024-12-04T15:44:52,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:444490x0, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-04T15:44:52,078 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44449-0x1017e2663e10000 connected
2024-12-04T15:44:52,097 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-04T15:44:52,097 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-04T15:44:52,098 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44449-0x1017e2663e10000, quorum=127.0.0.1:58881, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-04T15:44:52,098 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621, hbase.cluster.distributed=false
2024-12-04T15:44:52,100 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44449-0x1017e2663e10000, quorum=127.0.0.1:58881, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-04T15:44:52,100 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44449
2024-12-04T15:44:52,101 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44449
2024-12-04T15:44:52,101 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44449
2024-12-04T15:44:52,101 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44449
2024-12-04T15:44:52,101 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44449
2024-12-04T15:44:52,119 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/a21b6491b371:0 server-side Connection retries=45
2024-12-04T15:44:52,120 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-04T15:44:52,120 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-04T15:44:52,120 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-04T15:44:52,120 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-04T15:44:52,120 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-04T15:44:52,120 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-04T15:44:52,120 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-04T15:44:52,121 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39841
2024-12-04T15:44:52,123 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39841 connecting to ZooKeeper ensemble=127.0.0.1:58881
2024-12-04T15:44:52,124 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-04T15:44:52,124 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-04T15:44:52,127 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:398410x0, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-04T15:44:52,127 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39841-0x1017e2663e10001 connected
2024-12-04T15:44:52,127 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39841-0x1017e2663e10001, quorum=127.0.0.1:58881, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-04T15:44:52,128 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-04T15:44:52,129 INFO  [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/de2c1fb8-a351-04c3-cfcb-b73746afa260/bucket.cache
2024-12-04T15:44:52,129 INFO  [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false
2024-12-04T15:44:52,129 INFO  [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread
2024-12-04T15:44:52,129 INFO  [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100
2024-12-04T15:44:52,131 INFO  [Thread-118 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file
2024-12-04T15:44:52,131 WARN  [Thread-118 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache.
2024-12-04T15:44:52,131 INFO  [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/de2c1fb8-a351-04c3-cfcb-b73746afa260/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/de2c1fb8-a351-04c3-cfcb-b73746afa260/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator
2024-12-04T15:44:52,131 INFO  [Thread-118 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size))
2024-12-04T15:44:52,131 INFO  [Thread-118 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/de2c1fb8-a351-04c3-cfcb-b73746afa260/bucket.persistence is complete.
2024-12-04T15:44:52,131 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-04T15:44:52,132 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39841-0x1017e2663e10001, quorum=127.0.0.1:58881, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-04T15:44:52,133 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39841-0x1017e2663e10001, quorum=127.0.0.1:58881, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-04T15:44:52,135 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39841
2024-12-04T15:44:52,136 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39841
2024-12-04T15:44:52,140 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39841
2024-12-04T15:44:52,141 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39841
2024-12-04T15:44:52,141 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39841
2024-12-04T15:44:52,157 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/a21b6491b371:0 server-side Connection retries=45
2024-12-04T15:44:52,157 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-04T15:44:52,157 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-04T15:44:52,157 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-04T15:44:52,158 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-04T15:44:52,158 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-04T15:44:52,158 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-04T15:44:52,158 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-04T15:44:52,159 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34119
2024-12-04T15:44:52,160 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34119 connecting to ZooKeeper ensemble=127.0.0.1:58881
2024-12-04T15:44:52,160 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-04T15:44:52,160 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-04T15:44:52,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:341190x0, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-04T15:44:52,164 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34119-0x1017e2663e10002 connected
2024-12-04T15:44:52,164 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34119-0x1017e2663e10002, quorum=127.0.0.1:58881, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-04T15:44:52,164 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-04T15:44:52,166 INFO  [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/de2c1fb8-a351-04c3-cfcb-b73746afa260/bucket.cache
2024-12-04T15:44:52,166 INFO  [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false
2024-12-04T15:44:52,166 INFO  [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread
2024-12-04T15:44:52,166 INFO  [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100
2024-12-04T15:44:52,167 INFO  [Thread-119 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file
2024-12-04T15:44:52,167 WARN  [Thread-119 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache.
2024-12-04T15:44:52,167 INFO  [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/de2c1fb8-a351-04c3-cfcb-b73746afa260/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/de2c1fb8-a351-04c3-cfcb-b73746afa260/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator
2024-12-04T15:44:52,167 INFO  [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size))
2024-12-04T15:44:52,167 INFO  [Thread-119 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/de2c1fb8-a351-04c3-cfcb-b73746afa260/bucket.persistence is complete.
2024-12-04T15:44:52,167 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-04T15:44:52,168 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34119-0x1017e2663e10002, quorum=127.0.0.1:58881, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-12-04T15:44:52,169 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34119-0x1017e2663e10002, quorum=127.0.0.1:58881, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-04T15:44:52,173 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34119
2024-12-04T15:44:52,173 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34119
2024-12-04T15:44:52,176 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34119
2024-12-04T15:44:52,180 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34119
2024-12-04T15:44:52,180 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34119
2024-12-04T15:44:52,201 DEBUG [M:0;a21b6491b371:44449 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;a21b6491b371:44449
2024-12-04T15:44:52,201 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/a21b6491b371,44449,1733327092066
2024-12-04T15:44:52,203 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44449-0x1017e2663e10000, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-04T15:44:52,203 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39841-0x1017e2663e10001, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-04T15:44:52,203 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34119-0x1017e2663e10002, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-04T15:44:52,203 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44449-0x1017e2663e10000, quorum=127.0.0.1:58881, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/a21b6491b371,44449,1733327092066
2024-12-04T15:44:52,207 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39841-0x1017e2663e10001, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-04T15:44:52,207 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44449-0x1017e2663e10000, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-04T15:44:52,207 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34119-0x1017e2663e10002, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-12-04T15:44:52,207 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34119-0x1017e2663e10002, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-04T15:44:52,207 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39841-0x1017e2663e10001, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-04T15:44:52,208 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44449-0x1017e2663e10000, quorum=127.0.0.1:58881, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-04T15:44:52,208 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/a21b6491b371,44449,1733327092066 from backup master directory
2024-12-04T15:44:52,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44449-0x1017e2663e10000, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/a21b6491b371,44449,1733327092066
2024-12-04T15:44:52,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39841-0x1017e2663e10001, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-04T15:44:52,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44449-0x1017e2663e10000, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-04T15:44:52,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34119-0x1017e2663e10002, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-12-04T15:44:52,210 WARN  [master/a21b6491b371:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-04T15:44:52,210 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=a21b6491b371,44449,1733327092066
2024-12-04T15:44:52,212 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/hbase.id] with ID: 9b485e3f-9db5-4ebe-a775-c51db92356e7
2024-12-04T15:44:52,212 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/.tmp/hbase.id
2024-12-04T15:44:52,216 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/hbase.id]
2024-12-04T15:44:52,254 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-04T15:44:52,254 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-12-04T15:44:52,256 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-12-04T15:44:52,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44449-0x1017e2663e10000, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-04T15:44:52,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39841-0x1017e2663e10001, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-04T15:44:52,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34119-0x1017e2663e10002, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-04T15:44:52,264 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-04T15:44:52,265 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-12-04T15:44:52,265 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider
2024-12-04T15:44:52,286 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/MasterData/data/master/store
2024-12-04T15:44:52,292 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-04T15:44:52,292 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-04T15:44:52,292 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-04T15:44:52,292 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-04T15:44:52,292 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-04T15:44:52,292 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-04T15:44:52,292 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-04T15:44:52,292 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733327092292Disabling compacts and flushes for region at 1733327092292Disabling writes for close at 1733327092292Writing region close event to WAL at 1733327092292Closed at 1733327092292
2024-12-04T15:44:52,293 WARN  [master/a21b6491b371:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/MasterData/data/master/store/.initializing
2024-12-04T15:44:52,293 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/MasterData/WALs/a21b6491b371,44449,1733327092066
2024-12-04T15:44:52,298 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName
2024-12-04T15:44:52,302 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a21b6491b371%2C44449%2C1733327092066, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/MasterData/WALs/a21b6491b371,44449,1733327092066, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/MasterData/oldWALs, maxLogs=10
2024-12-04T15:44:52,320 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/MasterData/WALs/a21b6491b371,44449,1733327092066/a21b6491b371%2C44449%2C1733327092066.1733327092302
2024-12-04T15:44:52,321 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: []
2024-12-04T15:44:52,321 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-12-04T15:44:52,321 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-04T15:44:52,321 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-12-04T15:44:52,321 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-12-04T15:44:52,322 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-04T15:44:52,327 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-12-04T15:44:52,327 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-04T15:44:52,327 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-04T15:44:52,327 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-04T15:44:52,331 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-12-04T15:44:52,331 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-04T15:44:52,332 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-04T15:44:52,332 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-04T15:44:52,335 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-12-04T15:44:52,335 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-04T15:44:52,336 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-04T15:44:52,336 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-12-04T15:44:52,339 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-12-04T15:44:52,339 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-04T15:44:52,340 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-04T15:44:52,340 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-12-04T15:44:52,340 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-04T15:44:52,340 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-12-04T15:44:52,341 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-12-04T15:44:52,341 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-12-04T15:44:52,341 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-04T15:44:52,341 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-12-04T15:44:52,347 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-04T15:44:52,348 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73669023, jitterRate=0.09775398671627045}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-04T15:44:52,349 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1733327092321Initializing all the Stores at 1733327092322 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733327092322Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733327092322Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733327092322Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733327092322Cleaning up temporary data from old regions at 1733327092341 (+19 ms)Region opened successfully at 1733327092348 (+7 ms)
2024-12-04T15:44:52,349 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-12-04T15:44:52,353 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f252b44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a21b6491b371/172.17.0.2:0
2024-12-04T15:44:52,354 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-12-04T15:44:52,354 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-12-04T15:44:52,354 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-12-04T15:44:52,354 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-12-04T15:44:52,355 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-12-04T15:44:52,355 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-12-04T15:44:52,355 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-12-04T15:44:52,357 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-12-04T15:44:52,358 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44449-0x1017e2663e10000, quorum=127.0.0.1:58881, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-12-04T15:44:52,359 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-12-04T15:44:52,360 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-12-04T15:44:52,360 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44449-0x1017e2663e10000, quorum=127.0.0.1:58881, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-12-04T15:44:52,363 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-12-04T15:44:52,363 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-12-04T15:44:52,364 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44449-0x1017e2663e10000, quorum=127.0.0.1:58881, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-12-04T15:44:52,365 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-12-04T15:44:52,366 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44449-0x1017e2663e10000, quorum=127.0.0.1:58881, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-12-04T15:44:52,367 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-12-04T15:44:52,369 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44449-0x1017e2663e10000, quorum=127.0.0.1:58881, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-12-04T15:44:52,370 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-12-04T15:44:52,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44449-0x1017e2663e10000, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-04T15:44:52,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39841-0x1017e2663e10001, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-04T15:44:52,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34119-0x1017e2663e10002, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-12-04T15:44:52,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44449-0x1017e2663e10000, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-04T15:44:52,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39841-0x1017e2663e10001, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-04T15:44:52,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34119-0x1017e2663e10002, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-04T15:44:52,374 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=a21b6491b371,44449,1733327092066, sessionid=0x1017e2663e10000, setting cluster-up flag (Was=false)
2024-12-04T15:44:52,377 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44449-0x1017e2663e10000, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-04T15:44:52,377 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34119-0x1017e2663e10002, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-04T15:44:52,377 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39841-0x1017e2663e10001, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-04T15:44:52,382 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-12-04T15:44:52,384 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a21b6491b371,44449,1733327092066
2024-12-04T15:44:52,386 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34119-0x1017e2663e10002, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-04T15:44:52,386 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44449-0x1017e2663e10000, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-04T15:44:52,386 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39841-0x1017e2663e10001, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-04T15:44:52,392 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-12-04T15:44:52,394 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a21b6491b371,44449,1733327092066
2024-12-04T15:44:52,395 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-12-04T15:44:52,396 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-12-04T15:44:52,397 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-12-04T15:44:52,397 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-12-04T15:44:52,397 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: a21b6491b371,44449,1733327092066
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-12-04T15:44:52,399 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/a21b6491b371:0, corePoolSize=5, maxPoolSize=5
2024-12-04T15:44:52,399 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/a21b6491b371:0, corePoolSize=5, maxPoolSize=5
2024-12-04T15:44:52,399 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/a21b6491b371:0, corePoolSize=5, maxPoolSize=5
2024-12-04T15:44:52,399 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/a21b6491b371:0, corePoolSize=5, maxPoolSize=5
2024-12-04T15:44:52,399 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/a21b6491b371:0, corePoolSize=10, maxPoolSize=10
2024-12-04T15:44:52,399 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:52,399 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/a21b6491b371:0, corePoolSize=2, maxPoolSize=2
2024-12-04T15:44:52,399 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:52,400 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733327122400
2024-12-04T15:44:52,400 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-12-04T15:44:52,400 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-12-04T15:44:52,400 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-12-04T15:44:52,400 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-12-04T15:44:52,400 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-12-04T15:44:52,400 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-12-04T15:44:52,401 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-04T15:44:52,401 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-12-04T15:44:52,401 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:52,401 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-04T15:44:52,402 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-12-04T15:44:52,402 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-12-04T15:44:52,402 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-12-04T15:44:52,402 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-12-04T15:44:52,403 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-12-04T15:44:52,403 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-12-04T15:44:52,403 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/a21b6491b371:0:becomeActiveMaster-HFileCleaner.large.0-1733327092403,5,FailOnTimeoutGroup]
2024-12-04T15:44:52,404 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/a21b6491b371:0:becomeActiveMaster-HFileCleaner.small.0-1733327092403,5,FailOnTimeoutGroup]
2024-12-04T15:44:52,404 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:52,404 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-12-04T15:44:52,404 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:52,404 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:52,416 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-12-04T15:44:52,416 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621
2024-12-04T15:44:52,423 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-04T15:44:52,424 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 
2024-12-04T15:44:52,427 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-04T15:44:52,427 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-04T15:44:52,428 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-04T15:44:52,428 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 
2024-12-04T15:44:52,431 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-04T15:44:52,431 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-04T15:44:52,431 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-04T15:44:52,431 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 
2024-12-04T15:44:52,435 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-04T15:44:52,435 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-04T15:44:52,435 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-04T15:44:52,435 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 
2024-12-04T15:44:52,438 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-04T15:44:52,438 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-04T15:44:52,439 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-04T15:44:52,439 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-04T15:44:52,439 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740
2024-12-04T15:44:52,439 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740
2024-12-04T15:44:52,440 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-04T15:44:52,440 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-04T15:44:52,440 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-04T15:44:52,440 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-04T15:44:52,446 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-04T15:44:52,446 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69739190, jitterRate=0.03919491171836853}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-04T15:44:52,447 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1733327092423Initializing all the Stores at 1733327092423Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733327092424 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733327092424Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733327092424Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733327092424Cleaning up temporary data from old regions at 1733327092440 (+16 ms)Region opened successfully at 1733327092447 (+7 ms)
2024-12-04T15:44:52,447 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-04T15:44:52,447 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-04T15:44:52,447 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-04T15:44:52,447 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-04T15:44:52,447 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-04T15:44:52,448 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-04T15:44:52,448 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733327092447Disabling compacts and flushes for region at 1733327092447Disabling writes for close at 1733327092447Writing region close event to WAL at 1733327092448 (+1 ms)Closed at 1733327092448
2024-12-04T15:44:52,449 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-04T15:44:52,449 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-12-04T15:44:52,449 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-04T15:44:52,450 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-04T15:44:52,451 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-12-04T15:44:52,485 INFO  [RS:0;a21b6491b371:39841 {}] regionserver.HRegionServer(746): ClusterId : 9b485e3f-9db5-4ebe-a775-c51db92356e7
2024-12-04T15:44:52,485 INFO  [RS:1;a21b6491b371:34119 {}] regionserver.HRegionServer(746): ClusterId : 9b485e3f-9db5-4ebe-a775-c51db92356e7
2024-12-04T15:44:52,485 DEBUG [RS:1;a21b6491b371:34119 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-04T15:44:52,485 DEBUG [RS:0;a21b6491b371:39841 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-04T15:44:52,488 DEBUG [RS:1;a21b6491b371:34119 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-04T15:44:52,488 DEBUG [RS:0;a21b6491b371:39841 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-04T15:44:52,488 DEBUG [RS:0;a21b6491b371:39841 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-04T15:44:52,488 DEBUG [RS:1;a21b6491b371:34119 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-04T15:44:52,491 DEBUG [RS:0;a21b6491b371:39841 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-04T15:44:52,491 DEBUG [RS:1;a21b6491b371:34119 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-04T15:44:52,492 DEBUG [RS:0;a21b6491b371:39841 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1183ecfc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a21b6491b371/172.17.0.2:0
2024-12-04T15:44:52,492 DEBUG [RS:1;a21b6491b371:34119 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b4dc2f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a21b6491b371/172.17.0.2:0
2024-12-04T15:44:52,505 DEBUG [RS:1;a21b6491b371:34119 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;a21b6491b371:34119
2024-12-04T15:44:52,506 INFO  [RS:1;a21b6491b371:34119 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-04T15:44:52,506 INFO  [RS:1;a21b6491b371:34119 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-04T15:44:52,506 DEBUG [RS:1;a21b6491b371:34119 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-04T15:44:52,506 DEBUG [RS:0;a21b6491b371:39841 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;a21b6491b371:39841
2024-12-04T15:44:52,506 INFO  [RS:0;a21b6491b371:39841 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-04T15:44:52,506 INFO  [RS:0;a21b6491b371:39841 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-04T15:44:52,506 DEBUG [RS:0;a21b6491b371:39841 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-04T15:44:52,507 INFO  [RS:1;a21b6491b371:34119 {}] regionserver.HRegionServer(2659): reportForDuty to master=a21b6491b371,44449,1733327092066 with port=34119, startcode=1733327092157
2024-12-04T15:44:52,507 INFO  [RS:0;a21b6491b371:39841 {}] regionserver.HRegionServer(2659): reportForDuty to master=a21b6491b371,44449,1733327092066 with port=39841, startcode=1733327092119
2024-12-04T15:44:52,507 DEBUG [RS:0;a21b6491b371:39841 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-04T15:44:52,507 DEBUG [RS:1;a21b6491b371:34119 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-04T15:44:52,510 INFO  [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42711, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService
2024-12-04T15:44:52,510 INFO  [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40597, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService
2024-12-04T15:44:52,511 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44449 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a21b6491b371,39841,1733327092119
2024-12-04T15:44:52,511 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44449 {}] master.ServerManager(517): Registering regionserver=a21b6491b371,39841,1733327092119
2024-12-04T15:44:52,512 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44449 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a21b6491b371,34119,1733327092157
2024-12-04T15:44:52,512 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44449 {}] master.ServerManager(517): Registering regionserver=a21b6491b371,34119,1733327092157
2024-12-04T15:44:52,513 DEBUG [RS:0;a21b6491b371:39841 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621
2024-12-04T15:44:52,513 DEBUG [RS:0;a21b6491b371:39841 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/
2024-12-04T15:44:52,513 DEBUG [RS:0;a21b6491b371:39841 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-04T15:44:52,514 DEBUG [RS:1;a21b6491b371:34119 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621
2024-12-04T15:44:52,514 DEBUG [RS:1;a21b6491b371:34119 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/
2024-12-04T15:44:52,514 DEBUG [RS:1;a21b6491b371:34119 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-04T15:44:52,515 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44449-0x1017e2663e10000, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-04T15:44:52,521 DEBUG [RS:0;a21b6491b371:39841 {}] zookeeper.ZKUtil(111): regionserver:39841-0x1017e2663e10001, quorum=127.0.0.1:58881, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a21b6491b371,39841,1733327092119
2024-12-04T15:44:52,521 WARN  [RS:0;a21b6491b371:39841 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-04T15:44:52,521 INFO  [RS:0;a21b6491b371:39841 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider
2024-12-04T15:44:52,521 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a21b6491b371,39841,1733327092119]
2024-12-04T15:44:52,521 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a21b6491b371,34119,1733327092157]
2024-12-04T15:44:52,521 DEBUG [RS:0;a21b6491b371:39841 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/WALs/a21b6491b371,39841,1733327092119
2024-12-04T15:44:52,521 DEBUG [RS:1;a21b6491b371:34119 {}] zookeeper.ZKUtil(111): regionserver:34119-0x1017e2663e10002, quorum=127.0.0.1:58881, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a21b6491b371,34119,1733327092157
2024-12-04T15:44:52,521 WARN  [RS:1;a21b6491b371:34119 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-04T15:44:52,521 INFO  [RS:1;a21b6491b371:34119 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider
2024-12-04T15:44:52,521 DEBUG [RS:1;a21b6491b371:34119 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/WALs/a21b6491b371,34119,1733327092157
2024-12-04T15:44:52,527 INFO  [RS:1;a21b6491b371:34119 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-04T15:44:52,527 INFO  [RS:0;a21b6491b371:39841 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-04T15:44:52,529 INFO  [RS:1;a21b6491b371:34119 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-04T15:44:52,530 INFO  [RS:1;a21b6491b371:34119 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-04T15:44:52,530 INFO  [RS:1;a21b6491b371:34119 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:52,530 INFO  [RS:1;a21b6491b371:34119 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-04T15:44:52,530 INFO  [RS:1;a21b6491b371:34119 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-04T15:44:52,531 INFO  [RS:1;a21b6491b371:34119 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:52,531 DEBUG [RS:1;a21b6491b371:34119 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:52,531 DEBUG [RS:1;a21b6491b371:34119 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:52,531 DEBUG [RS:1;a21b6491b371:34119 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:52,531 DEBUG [RS:1;a21b6491b371:34119 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:52,531 DEBUG [RS:1;a21b6491b371:34119 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:52,531 DEBUG [RS:1;a21b6491b371:34119 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a21b6491b371:0, corePoolSize=2, maxPoolSize=2
2024-12-04T15:44:52,531 DEBUG [RS:1;a21b6491b371:34119 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:52,531 DEBUG [RS:1;a21b6491b371:34119 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:52,531 DEBUG [RS:1;a21b6491b371:34119 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:52,531 DEBUG [RS:1;a21b6491b371:34119 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:52,532 DEBUG [RS:1;a21b6491b371:34119 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:52,532 DEBUG [RS:1;a21b6491b371:34119 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:52,532 DEBUG [RS:1;a21b6491b371:34119 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a21b6491b371:0, corePoolSize=3, maxPoolSize=3
2024-12-04T15:44:52,532 DEBUG [RS:1;a21b6491b371:34119 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a21b6491b371:0, corePoolSize=3, maxPoolSize=3
2024-12-04T15:44:52,532 INFO  [RS:0;a21b6491b371:39841 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-04T15:44:52,533 INFO  [RS:1;a21b6491b371:34119 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:52,533 INFO  [RS:1;a21b6491b371:34119 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:52,533 INFO  [RS:0;a21b6491b371:39841 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-04T15:44:52,533 INFO  [RS:1;a21b6491b371:34119 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:52,533 INFO  [RS:1;a21b6491b371:34119 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:52,533 INFO  [RS:0;a21b6491b371:39841 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:52,533 INFO  [RS:1;a21b6491b371:34119 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:52,533 INFO  [RS:1;a21b6491b371:34119 {}] hbase.ChoreService(168): Chore ScheduledChore name=a21b6491b371,34119,1733327092157-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-04T15:44:52,536 INFO  [RS:0;a21b6491b371:39841 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-04T15:44:52,537 INFO  [RS:0;a21b6491b371:39841 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-04T15:44:52,537 INFO  [RS:0;a21b6491b371:39841 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:52,537 DEBUG [RS:0;a21b6491b371:39841 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:52,538 DEBUG [RS:0;a21b6491b371:39841 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:52,538 DEBUG [RS:0;a21b6491b371:39841 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:52,538 DEBUG [RS:0;a21b6491b371:39841 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:52,538 DEBUG [RS:0;a21b6491b371:39841 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:52,538 DEBUG [RS:0;a21b6491b371:39841 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a21b6491b371:0, corePoolSize=2, maxPoolSize=2
2024-12-04T15:44:52,538 DEBUG [RS:0;a21b6491b371:39841 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:52,538 DEBUG [RS:0;a21b6491b371:39841 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:52,538 DEBUG [RS:0;a21b6491b371:39841 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:52,538 DEBUG [RS:0;a21b6491b371:39841 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:52,538 DEBUG [RS:0;a21b6491b371:39841 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:52,539 DEBUG [RS:0;a21b6491b371:39841 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:52,539 DEBUG [RS:0;a21b6491b371:39841 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a21b6491b371:0, corePoolSize=3, maxPoolSize=3
2024-12-04T15:44:52,539 DEBUG [RS:0;a21b6491b371:39841 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a21b6491b371:0, corePoolSize=3, maxPoolSize=3
2024-12-04T15:44:52,539 INFO  [RS:0;a21b6491b371:39841 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:52,539 INFO  [RS:0;a21b6491b371:39841 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:52,539 INFO  [RS:0;a21b6491b371:39841 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:52,539 INFO  [RS:0;a21b6491b371:39841 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:52,540 INFO  [RS:0;a21b6491b371:39841 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:52,540 INFO  [RS:0;a21b6491b371:39841 {}] hbase.ChoreService(168): Chore ScheduledChore name=a21b6491b371,39841,1733327092119-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-04T15:44:52,555 INFO  [RS:1;a21b6491b371:34119 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-04T15:44:52,555 INFO  [RS:0;a21b6491b371:39841 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-04T15:44:52,555 INFO  [RS:0;a21b6491b371:39841 {}] hbase.ChoreService(168): Chore ScheduledChore name=a21b6491b371,39841,1733327092119-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:52,555 INFO  [RS:1;a21b6491b371:34119 {}] hbase.ChoreService(168): Chore ScheduledChore name=a21b6491b371,34119,1733327092157-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:52,555 INFO  [RS:0;a21b6491b371:39841 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:52,555 INFO  [RS:1;a21b6491b371:34119 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:52,555 INFO  [RS:0;a21b6491b371:39841 {}] regionserver.Replication(171): a21b6491b371,39841,1733327092119 started
2024-12-04T15:44:52,555 INFO  [RS:1;a21b6491b371:34119 {}] regionserver.Replication(171): a21b6491b371,34119,1733327092157 started
2024-12-04T15:44:52,569 INFO  [RS:0;a21b6491b371:39841 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:52,570 INFO  [RS:1;a21b6491b371:34119 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:52,570 INFO  [RS:0;a21b6491b371:39841 {}] regionserver.HRegionServer(1482): Serving as a21b6491b371,39841,1733327092119, RpcServer on a21b6491b371/172.17.0.2:39841, sessionid=0x1017e2663e10001
2024-12-04T15:44:52,570 INFO  [RS:1;a21b6491b371:34119 {}] regionserver.HRegionServer(1482): Serving as a21b6491b371,34119,1733327092157, RpcServer on a21b6491b371/172.17.0.2:34119, sessionid=0x1017e2663e10002
2024-12-04T15:44:52,570 DEBUG [RS:1;a21b6491b371:34119 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-04T15:44:52,570 DEBUG [RS:0;a21b6491b371:39841 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-04T15:44:52,570 DEBUG [RS:1;a21b6491b371:34119 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a21b6491b371,34119,1733327092157
2024-12-04T15:44:52,570 DEBUG [RS:0;a21b6491b371:39841 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a21b6491b371,39841,1733327092119
2024-12-04T15:44:52,570 DEBUG [RS:1;a21b6491b371:34119 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a21b6491b371,34119,1733327092157'
2024-12-04T15:44:52,570 DEBUG [RS:0;a21b6491b371:39841 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a21b6491b371,39841,1733327092119'
2024-12-04T15:44:52,570 DEBUG [RS:0;a21b6491b371:39841 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-04T15:44:52,570 DEBUG [RS:1;a21b6491b371:34119 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-04T15:44:52,571 DEBUG [RS:1;a21b6491b371:34119 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-04T15:44:52,571 DEBUG [RS:0;a21b6491b371:39841 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-04T15:44:52,571 DEBUG [RS:1;a21b6491b371:34119 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-04T15:44:52,571 DEBUG [RS:0;a21b6491b371:39841 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-04T15:44:52,571 DEBUG [RS:1;a21b6491b371:34119 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-04T15:44:52,571 DEBUG [RS:0;a21b6491b371:39841 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-04T15:44:52,571 DEBUG [RS:1;a21b6491b371:34119 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a21b6491b371,34119,1733327092157
2024-12-04T15:44:52,571 DEBUG [RS:0;a21b6491b371:39841 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a21b6491b371,39841,1733327092119
2024-12-04T15:44:52,571 DEBUG [RS:1;a21b6491b371:34119 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a21b6491b371,34119,1733327092157'
2024-12-04T15:44:52,571 DEBUG [RS:0;a21b6491b371:39841 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a21b6491b371,39841,1733327092119'
2024-12-04T15:44:52,571 DEBUG [RS:1;a21b6491b371:34119 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-04T15:44:52,571 DEBUG [RS:0;a21b6491b371:39841 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-04T15:44:52,572 DEBUG [RS:1;a21b6491b371:34119 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-04T15:44:52,572 DEBUG [RS:0;a21b6491b371:39841 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-04T15:44:52,572 DEBUG [RS:1;a21b6491b371:34119 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-04T15:44:52,573 INFO  [RS:1;a21b6491b371:34119 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-04T15:44:52,573 DEBUG [RS:0;a21b6491b371:39841 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-04T15:44:52,573 INFO  [RS:1;a21b6491b371:34119 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-04T15:44:52,573 INFO  [RS:0;a21b6491b371:39841 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-04T15:44:52,573 INFO  [RS:0;a21b6491b371:39841 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-04T15:44:52,601 WARN  [a21b6491b371:44449 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions.
2024-12-04T15:44:52,673 INFO  [RS:0;a21b6491b371:39841 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName
2024-12-04T15:44:52,673 INFO  [RS:1;a21b6491b371:34119 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName
2024-12-04T15:44:52,674 INFO  [RS:1;a21b6491b371:34119 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=a21b6491b371%2C34119%2C1733327092157, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/WALs/a21b6491b371,34119,1733327092157, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/oldWALs, maxLogs=55
2024-12-04T15:44:52,674 INFO  [RS:0;a21b6491b371:39841 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=a21b6491b371%2C39841%2C1733327092119, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/WALs/a21b6491b371,39841,1733327092119, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/oldWALs, maxLogs=55
2024-12-04T15:44:52,695 INFO  [RS:1;a21b6491b371:34119 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/WALs/a21b6491b371,34119,1733327092157/a21b6491b371%2C34119%2C1733327092157.1733327092677
2024-12-04T15:44:52,695 INFO  [RS:0;a21b6491b371:39841 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/WALs/a21b6491b371,39841,1733327092119/a21b6491b371%2C39841%2C1733327092119.1733327092677
2024-12-04T15:44:52,695 DEBUG [RS:1;a21b6491b371:34119 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: []
2024-12-04T15:44:52,695 DEBUG [RS:0;a21b6491b371:39841 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: []
2024-12-04T15:44:52,851 DEBUG [a21b6491b371:44449 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2
2024-12-04T15:44:52,852 DEBUG [a21b6491b371:44449 {}] balancer.BalancerClusterState(204): Hosts are {a21b6491b371=0} racks are {/default-rack=0}
2024-12-04T15:44:52,852 DEBUG [a21b6491b371:44449 {}] balancer.BalancerClusterState(303): server 0 has 0 regions
2024-12-04T15:44:52,852 DEBUG [a21b6491b371:44449 {}] balancer.BalancerClusterState(303): server 1 has 0 regions
2024-12-04T15:44:52,852 DEBUG [a21b6491b371:44449 {}] balancer.BalancerClusterState(310): server 0 is on host 0
2024-12-04T15:44:52,852 DEBUG [a21b6491b371:44449 {}] balancer.BalancerClusterState(310): server 1 is on host 0
2024-12-04T15:44:52,852 INFO  [a21b6491b371:44449 {}] balancer.BalancerClusterState(321): server 0 is on rack 0
2024-12-04T15:44:52,852 INFO  [a21b6491b371:44449 {}] balancer.BalancerClusterState(321): server 1 is on rack 0
2024-12-04T15:44:52,852 DEBUG [a21b6491b371:44449 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1
2024-12-04T15:44:52,853 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a21b6491b371,39841,1733327092119
2024-12-04T15:44:52,854 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a21b6491b371,39841,1733327092119, state=OPENING
2024-12-04T15:44:52,857 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-12-04T15:44:52,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44449-0x1017e2663e10000, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-04T15:44:52,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39841-0x1017e2663e10001, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-04T15:44:52,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34119-0x1017e2663e10002, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-04T15:44:52,859 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-04T15:44:52,859 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-04T15:44:52,859 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-04T15:44:52,859 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a21b6491b371,39841,1733327092119}]
2024-12-04T15:44:52,860 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-04T15:44:53,012 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-04T15:44:53,014 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42285, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-04T15:44:53,017 INFO  [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-04T15:44:53,017 INFO  [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider
2024-12-04T15:44:53,017 INFO  [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta
2024-12-04T15:44:53,018 INFO  [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=a21b6491b371%2C39841%2C1733327092119.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/WALs/a21b6491b371,39841,1733327092119, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/oldWALs, maxLogs=55
2024-12-04T15:44:53,036 INFO  [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/WALs/a21b6491b371,39841,1733327092119/a21b6491b371%2C39841%2C1733327092119.meta.1733327093019.meta
2024-12-04T15:44:53,037 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: []
2024-12-04T15:44:53,037 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-04T15:44:53,037 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-04T15:44:53,037 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-04T15:44:53,037 INFO  [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-04T15:44:53,038 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-04T15:44:53,038 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-04T15:44:53,038 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-04T15:44:53,038 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-04T15:44:53,039 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 
2024-12-04T15:44:53,039 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-04T15:44:53,040 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-04T15:44:53,040 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-04T15:44:53,040 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 
2024-12-04T15:44:53,041 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-04T15:44:53,041 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-04T15:44:53,041 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-04T15:44:53,041 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 
2024-12-04T15:44:53,041 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-04T15:44:53,041 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-04T15:44:53,042 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-04T15:44:53,042 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 
2024-12-04T15:44:53,042 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-04T15:44:53,042 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-04T15:44:53,042 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-04T15:44:53,043 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-04T15:44:53,043 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740
2024-12-04T15:44:53,043 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740
2024-12-04T15:44:53,044 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-04T15:44:53,044 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-04T15:44:53,044 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-04T15:44:53,045 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-04T15:44:53,045 INFO  [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66495260, jitterRate=-0.009143412113189697}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-04T15:44:53,045 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-04T15:44:53,046 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733327093038Writing region info on filesystem at 1733327093038Initializing all the Stores at 1733327093038Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733327093038Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733327093039 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733327093039Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733327093039Cleaning up temporary data from old regions at 1733327093044 (+5 ms)Running coprocessor post-open hooks at 1733327093045 (+1 ms)Region opened successfully at 1733327093046 (+1 ms)
2024-12-04T15:44:53,046 INFO  [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733327093012
2024-12-04T15:44:53,049 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-04T15:44:53,049 INFO  [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-04T15:44:53,049 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=a21b6491b371,39841,1733327092119
2024-12-04T15:44:53,050 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a21b6491b371,39841,1733327092119, state=OPEN
2024-12-04T15:44:53,052 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44449-0x1017e2663e10000, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-04T15:44:53,052 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39841-0x1017e2663e10001, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-04T15:44:53,052 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34119-0x1017e2663e10002, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-04T15:44:53,053 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-04T15:44:53,053 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a21b6491b371,39841,1733327092119
2024-12-04T15:44:53,053 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-04T15:44:53,053 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-04T15:44:53,055 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-12-04T15:44:53,055 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a21b6491b371,39841,1733327092119 in 194 msec
2024-12-04T15:44:53,056 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-12-04T15:44:53,056 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 606 msec
2024-12-04T15:44:53,057 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-12-04T15:44:53,057 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-12-04T15:44:53,060 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-04T15:44:53,060 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a21b6491b371,39841,1733327092119, seqNum=-1]
2024-12-04T15:44:53,061 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-04T15:44:53,063 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49919, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-04T15:44:53,067 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 671 msec
2024-12-04T15:44:53,067 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733327093067, completionTime=-1
2024-12-04T15:44:53,067 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running
2024-12-04T15:44:53,068 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster...
2024-12-04T15:44:53,069 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2
2024-12-04T15:44:53,070 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733327153070
2024-12-04T15:44:53,070 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733327213070
2024-12-04T15:44:53,070 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec
2024-12-04T15:44:53,070 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a21b6491b371,44449,1733327092066-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:53,070 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a21b6491b371,44449,1733327092066-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:53,070 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a21b6491b371,44449,1733327092066-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:53,070 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-a21b6491b371:44449, period=300000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:53,070 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:53,071 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:53,073 DEBUG [master/a21b6491b371:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-12-04T15:44:53,075 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.865sec
2024-12-04T15:44:53,075 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-12-04T15:44:53,075 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-12-04T15:44:53,075 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-12-04T15:44:53,075 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-12-04T15:44:53,075 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-12-04T15:44:53,076 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a21b6491b371,44449,1733327092066-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-04T15:44:53,076 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a21b6491b371,44449,1733327092066-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-12-04T15:44:53,076 DEBUG [master/a21b6491b371:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-12-04T15:44:53,077 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-12-04T15:44:53,077 INFO  [master/a21b6491b371:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a21b6491b371,44449,1733327092066-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:53,085 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7559f21a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-04T15:44:53,085 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a21b6491b371,44449,-1 for getting cluster id
2024-12-04T15:44:53,085 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-12-04T15:44:53,087 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9b485e3f-9db5-4ebe-a775-c51db92356e7'
2024-12-04T15:44:53,087 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-12-04T15:44:53,088 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9b485e3f-9db5-4ebe-a775-c51db92356e7"

2024-12-04T15:44:53,088 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ed63546, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-04T15:44:53,088 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a21b6491b371,44449,-1]
2024-12-04T15:44:53,088 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-12-04T15:44:53,089 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-04T15:44:53,090 INFO  [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55094, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-12-04T15:44:53,091 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27316b7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-12-04T15:44:53,092 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-04T15:44:53,093 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a21b6491b371,39841,1733327092119, seqNum=-1]
2024-12-04T15:44:53,093 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-04T15:44:53,096 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56876, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-04T15:44:53,098 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=a21b6491b371,44449,1733327092066
2024-12-04T15:44:53,098 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-12-04T15:44:53,099 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.AsyncConnectionImpl(321): The fetched master address is a21b6491b371,44449,1733327092066
2024-12-04T15:44:53,099 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@c57bde7
2024-12-04T15:44:53,099 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-12-04T15:44:53,101 INFO  [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55108, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-12-04T15:44:53,102 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44449 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1733327093098', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-12-04T15:44:53,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44449 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733327093098
2024-12-04T15:44:53,104 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733327093098 execute state=CREATE_TABLE_PRE_OPERATION
2024-12-04T15:44:53,104 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-04T15:44:53,104 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44449 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default"
qualifier: "testBlockEvictionOnGracefulStop1733327093098"
 procId is: 4
2024-12-04T15:44:53,105 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733327093098 execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-12-04T15:44:53,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-04T15:44:53,119 INFO  [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733327093098-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => cee5398235ad5e94b1f3bbb5f1071cfe, NAME => 'testBlockEvictionOnGracefulStop1733327093098,,1733327093101.cee5398235ad5e94b1f3bbb5f1071cfe.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1733327093098', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621
2024-12-04T15:44:53,128 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733327093098-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733327093098,,1733327093101.cee5398235ad5e94b1f3bbb5f1071cfe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-04T15:44:53,128 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733327093098-pool-0 {}] regionserver.HRegion(1722): Closing cee5398235ad5e94b1f3bbb5f1071cfe, disabling compactions & flushes
2024-12-04T15:44:53,128 INFO  [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733327093098-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733327093098,,1733327093101.cee5398235ad5e94b1f3bbb5f1071cfe.
2024-12-04T15:44:53,128 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733327093098-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733327093098,,1733327093101.cee5398235ad5e94b1f3bbb5f1071cfe.
2024-12-04T15:44:53,128 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733327093098-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733327093098,,1733327093101.cee5398235ad5e94b1f3bbb5f1071cfe. after waiting 0 ms
2024-12-04T15:44:53,128 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733327093098-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733327093098,,1733327093101.cee5398235ad5e94b1f3bbb5f1071cfe.
2024-12-04T15:44:53,128 INFO  [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733327093098-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733327093098,,1733327093101.cee5398235ad5e94b1f3bbb5f1071cfe.
2024-12-04T15:44:53,129 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733327093098-pool-0 {}] regionserver.HRegion(1676): Region close journal for cee5398235ad5e94b1f3bbb5f1071cfe:
Waiting for close lock at 1733327093128Disabling compacts and flushes for region at 1733327093128Disabling writes for close at 1733327093128Writing region close event to WAL at 1733327093128Closed at 1733327093128
2024-12-04T15:44:53,130 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733327093098 execute state=CREATE_TABLE_ADD_TO_META
2024-12-04T15:44:53,130 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1733327093098,,1733327093101.cee5398235ad5e94b1f3bbb5f1071cfe.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1733327093130"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733327093130"}]},"ts":"1733327093130"}
2024-12-04T15:44:53,133 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-12-04T15:44:53,134 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733327093098 execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-12-04T15:44:53,134 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1733327093098","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733327093134"}]},"ts":"1733327093134"}
2024-12-04T15:44:53,136 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1733327093098, state=ENABLING in hbase:meta
2024-12-04T15:44:53,137 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {a21b6491b371=0} racks are {/default-rack=0}
2024-12-04T15:44:53,137 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions
2024-12-04T15:44:53,137 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions
2024-12-04T15:44:53,137 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0
2024-12-04T15:44:53,137 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0
2024-12-04T15:44:53,137 INFO  [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0
2024-12-04T15:44:53,137 INFO  [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0
2024-12-04T15:44:53,137 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1
2024-12-04T15:44:53,137 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733327093098, region=cee5398235ad5e94b1f3bbb5f1071cfe, ASSIGN}]
2024-12-04T15:44:53,138 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733327093098, region=cee5398235ad5e94b1f3bbb5f1071cfe, ASSIGN
2024-12-04T15:44:53,139 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733327093098, region=cee5398235ad5e94b1f3bbb5f1071cfe, ASSIGN; state=OFFLINE, location=a21b6491b371,39841,1733327092119; forceNewPlan=false, retain=false
2024-12-04T15:44:53,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-04T15:44:53,290 INFO  [a21b6491b371:44449 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 
2024-12-04T15:44:53,290 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=cee5398235ad5e94b1f3bbb5f1071cfe, regionState=OPENING, regionLocation=a21b6491b371,39841,1733327092119
2024-12-04T15:44:53,294 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733327093098, region=cee5398235ad5e94b1f3bbb5f1071cfe, ASSIGN because future has completed
2024-12-04T15:44:53,294 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure cee5398235ad5e94b1f3bbb5f1071cfe, server=a21b6491b371,39841,1733327092119}]
2024-12-04T15:44:53,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-04T15:44:53,449 INFO  [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1733327093098,,1733327093101.cee5398235ad5e94b1f3bbb5f1071cfe.
2024-12-04T15:44:53,449 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => cee5398235ad5e94b1f3bbb5f1071cfe, NAME => 'testBlockEvictionOnGracefulStop1733327093098,,1733327093101.cee5398235ad5e94b1f3bbb5f1071cfe.', STARTKEY => '', ENDKEY => ''}
2024-12-04T15:44:53,449 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1733327093098 cee5398235ad5e94b1f3bbb5f1071cfe
2024-12-04T15:44:53,450 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733327093098,,1733327093101.cee5398235ad5e94b1f3bbb5f1071cfe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-04T15:44:53,450 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for cee5398235ad5e94b1f3bbb5f1071cfe
2024-12-04T15:44:53,450 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for cee5398235ad5e94b1f3bbb5f1071cfe
2024-12-04T15:44:53,451 INFO  [StoreOpener-cee5398235ad5e94b1f3bbb5f1071cfe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region cee5398235ad5e94b1f3bbb5f1071cfe 
2024-12-04T15:44:53,455 INFO  [StoreOpener-cee5398235ad5e94b1f3bbb5f1071cfe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cee5398235ad5e94b1f3bbb5f1071cfe columnFamilyName family
2024-12-04T15:44:53,455 DEBUG [StoreOpener-cee5398235ad5e94b1f3bbb5f1071cfe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-04T15:44:53,455 INFO  [StoreOpener-cee5398235ad5e94b1f3bbb5f1071cfe-1 {}] regionserver.HStore(327): Store=cee5398235ad5e94b1f3bbb5f1071cfe/family,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-04T15:44:53,456 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for cee5398235ad5e94b1f3bbb5f1071cfe
2024-12-04T15:44:53,456 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/default/testBlockEvictionOnGracefulStop1733327093098/cee5398235ad5e94b1f3bbb5f1071cfe
2024-12-04T15:44:53,456 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/default/testBlockEvictionOnGracefulStop1733327093098/cee5398235ad5e94b1f3bbb5f1071cfe
2024-12-04T15:44:53,457 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for cee5398235ad5e94b1f3bbb5f1071cfe
2024-12-04T15:44:53,457 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for cee5398235ad5e94b1f3bbb5f1071cfe
2024-12-04T15:44:53,457 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for cee5398235ad5e94b1f3bbb5f1071cfe
2024-12-04T15:44:53,464 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/default/testBlockEvictionOnGracefulStop1733327093098/cee5398235ad5e94b1f3bbb5f1071cfe/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-12-04T15:44:53,464 INFO  [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened cee5398235ad5e94b1f3bbb5f1071cfe; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69844388, jitterRate=0.040762484073638916}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-04T15:44:53,464 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cee5398235ad5e94b1f3bbb5f1071cfe
2024-12-04T15:44:53,465 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for cee5398235ad5e94b1f3bbb5f1071cfe:
Running coprocessor pre-open hook at 1733327093450Writing region info on filesystem at 1733327093450Initializing all the Stores at 1733327093450Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733327093450Cleaning up temporary data from old regions at 1733327093457 (+7 ms)Running coprocessor post-open hooks at 1733327093464 (+7 ms)Region opened successfully at 1733327093465 (+1 ms)
2024-12-04T15:44:53,465 INFO  [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1733327093098,,1733327093101.cee5398235ad5e94b1f3bbb5f1071cfe., pid=6, masterSystemTime=1733327093446
2024-12-04T15:44:53,468 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1733327093098,,1733327093101.cee5398235ad5e94b1f3bbb5f1071cfe.
2024-12-04T15:44:53,468 INFO  [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1733327093098,,1733327093101.cee5398235ad5e94b1f3bbb5f1071cfe.
2024-12-04T15:44:53,468 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=cee5398235ad5e94b1f3bbb5f1071cfe, regionState=OPEN, openSeqNum=2, regionLocation=a21b6491b371,39841,1733327092119
2024-12-04T15:44:53,471 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure cee5398235ad5e94b1f3bbb5f1071cfe, server=a21b6491b371,39841,1733327092119 because future has completed
2024-12-04T15:44:53,474 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-12-04T15:44:53,474 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure cee5398235ad5e94b1f3bbb5f1071cfe, server=a21b6491b371,39841,1733327092119 in 179 msec
2024-12-04T15:44:53,476 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-12-04T15:44:53,476 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733327093098, region=cee5398235ad5e94b1f3bbb5f1071cfe, ASSIGN in 337 msec
2024-12-04T15:44:53,477 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733327093098 execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-12-04T15:44:53,477 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1733327093098","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733327093477"}]},"ts":"1733327093477"}
2024-12-04T15:44:53,479 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1733327093098, state=ENABLED in hbase:meta
2024-12-04T15:44:53,479 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733327093098 execute state=CREATE_TABLE_POST_OPERATION
2024-12-04T15:44:53,481 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733327093098 in 378 msec
2024-12-04T15:44:53,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44449 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-12-04T15:44:53,733 INFO  [RPCClient-NioEventLoopGroup-5-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1733327093098 completed
2024-12-04T15:44:53,733 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1733327093098 get assigned. Timeout = 60000ms
2024-12-04T15:44:53,733 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1])
2024-12-04T15:44:53,737 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1733327093098 assigned to meta. Checking AM states.
2024-12-04T15:44:53,737 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1])
2024-12-04T15:44:53,737 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1733327093098 assigned.
2024-12-04T15:44:53,740 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1733327093098', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1733327093098,,1733327093101.cee5398235ad5e94b1f3bbb5f1071cfe., hostname=a21b6491b371,39841,1733327092119, seqNum=2]
2024-12-04T15:44:53,745 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing cee5398235ad5e94b1f3bbb5f1071cfe 1/1 column families, dataSize=78 B heapSize=480 B
2024-12-04T15:44:53,755 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b36d40b7dadc4cf0b4ca854bb9c05698
2024-12-04T15:44:53,769 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/default/testBlockEvictionOnGracefulStop1733327093098/cee5398235ad5e94b1f3bbb5f1071cfe/.tmp/family/b36d40b7dadc4cf0b4ca854bb9c05698 is 43, key is row1/family:qf1/1/Put/seqid=0
2024-12-04T15:44:53,769 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/default/testBlockEvictionOnGracefulStop1733327093098/cee5398235ad5e94b1f3bbb5f1071cfe/.tmp/family/b36d40b7dadc4cf0b4ca854bb9c05698
2024-12-04T15:44:53,772 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/default/testBlockEvictionOnGracefulStop1733327093098/cee5398235ad5e94b1f3bbb5f1071cfe/.tmp/family/b36d40b7dadc4cf0b4ca854bb9c05698 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/default/testBlockEvictionOnGracefulStop1733327093098/cee5398235ad5e94b1f3bbb5f1071cfe/family/b36d40b7dadc4cf0b4ca854bb9c05698
2024-12-04T15:44:53,773 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/default/testBlockEvictionOnGracefulStop1733327093098/cee5398235ad5e94b1f3bbb5f1071cfe/family/b36d40b7dadc4cf0b4ca854bb9c05698, delay=915 ms
2024-12-04T15:44:53,774 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/default/testBlockEvictionOnGracefulStop1733327093098/cee5398235ad5e94b1f3bbb5f1071cfe/family/b36d40b7dadc4cf0b4ca854bb9c05698, entries=2, sequenceid=6, filesize=4.9 K
2024-12-04T15:44:53,775 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for cee5398235ad5e94b1f3bbb5f1071cfe in 30ms, sequenceid=6, compaction requested=false
2024-12-04T15:44:53,775 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1733327093098'
2024-12-04T15:44:53,776 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for cee5398235ad5e94b1f3bbb5f1071cfe:

2024-12-04T15:44:54,377 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-12-04T15:44:54,689 DEBUG [hfile-prefetch-1733327091495 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b36d40b7dadc4cf0b4ca854bb9c05698_0. 
2024-12-04T15:44:54,690 DEBUG [hfile-prefetch-1733327091495 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b36d40b7dadc4cf0b4ca854bb9c05698_117. 
2024-12-04T15:44:54,690 DEBUG [hfile-prefetch-1733327091495 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/default/testBlockEvictionOnGracefulStop1733327093098/cee5398235ad5e94b1f3bbb5f1071cfe/family/b36d40b7dadc4cf0b4ca854bb9c05698, with total blocks 2, and data blocks 1
2024-12-04T15:44:54,690 DEBUG [hfile-prefetch-1733327091495 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map
2024-12-04T15:44:54,690 DEBUG [hfile-prefetch-1733327091495 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/default/testBlockEvictionOnGracefulStop1733327093098/cee5398235ad5e94b1f3bbb5f1071cfe/family/b36d40b7dadc4cf0b4ca854bb9c05698, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue.
2024-12-04T15:44:54,691 DEBUG [hfile-prefetch-1733327091495 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/default/testBlockEvictionOnGracefulStop1733327093098/cee5398235ad5e94b1f3bbb5f1071cfe/family/b36d40b7dadc4cf0b4ca854bb9c05698. Will sleep 100ms and try the verification again.
2024-12-04T15:44:54,731 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk
2024-12-04T15:44:54,731 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file
2024-12-04T15:44:54,731 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format.
2024-12-04T15:44:54,731 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000
2024-12-04T15:44:54,739 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0
2024-12-04T15:44:54,739 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1733327094731
2024-12-04T15:44:54,739 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming
2024-12-04T15:44:54,776 INFO  [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;a21b6491b371:39841,5,FailOnTimeoutGroup]
2024-12-04T15:44:54,776 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a21b6491b371,39841,1733327092119' *****
2024-12-04T15:44:54,776 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0
2024-12-04T15:44:54,777 INFO  [RS:0;a21b6491b371:39841 {}] bucket.BucketCache(1738): Disabling cache
2024-12-04T15:44:54,777 INFO  [RS:0;a21b6491b371:39841 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/de2c1fb8-a351-04c3-cfcb-b73746afa260/bucket.persistence
2024-12-04T15:44:54,777 INFO  [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false
2024-12-04T15:44:54,777 INFO  [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false
2024-12-04T15:44:54,777 INFO  [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false
2024-12-04T15:44:54,777 INFO  [RS:0;a21b6491b371:39841 {}] bucket.BucketCache(1768): Shutting down cache persister thread.
2024-12-04T15:44:54,777 WARN  [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread.
java.lang.InterruptedException: sleep interrupted
	at java.lang.Thread.sleep(Native Method) ~[?:?]
	at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?]
2024-12-04T15:44:54,787 DEBUG [RS:0;a21b6491b371:39841 {}] bucket.BucketCache(1487): Thread RS:0;a21b6491b371:39841 started persisting bucket cache to file
2024-12-04T15:44:54,787 DEBUG [RS:0;a21b6491b371:39841 {}] bucket.BucketCache(1494): Persist in new chunked persistence format.
2024-12-04T15:44:54,787 DEBUG [RS:0;a21b6491b371:39841 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000
2024-12-04T15:44:54,791 DEBUG [hfile-prefetch-1733327091495 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/default/testBlockEvictionOnGracefulStop1733327093098/cee5398235ad5e94b1f3bbb5f1071cfe/family/b36d40b7dadc4cf0b4ca854bb9c05698, with total blocks 2, and data blocks 1
2024-12-04T15:44:54,791 DEBUG [hfile-prefetch-1733327091495 {}] bucket.BucketCache(2285): iterating over 1 entries in the backing map
2024-12-04T15:44:54,791 DEBUG [RS:0;a21b6491b371:39841 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0
2024-12-04T15:44:54,791 DEBUG [RS:0;a21b6491b371:39841 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1733327094787
2024-12-04T15:44:54,791 DEBUG [hfile-prefetch-1733327091495 {}] bucket.BucketCache(2293): found block for file b36d40b7dadc4cf0b4ca854bb9c05698 in the backing map. Acquiring read lock for offset 0
2024-12-04T15:44:54,791 DEBUG [RS:0;a21b6491b371:39841 {}] bucket.BucketCache(1509): Thread RS:0;a21b6491b371:39841 finished persisting bucket cache to file, renaming
2024-12-04T15:44:54,791 DEBUG [hfile-prefetch-1733327091495 {}] bucket.BucketCache(2305): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/default/testBlockEvictionOnGracefulStop1733327093098/cee5398235ad5e94b1f3bbb5f1071cfe/family/b36d40b7dadc4cf0b4ca854bb9c05698 has now been fully cached.
2024-12-04T15:44:54,791 INFO  [RS:0;a21b6491b371:39841 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-04T15:44:54,791 INFO  [RS:0;a21b6491b371:39841 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-04T15:44:54,791 DEBUG [hfile-prefetch-1733327091495 {}] hfile.PrefetchExecutor(121): Prefetch completed for b36d40b7dadc4cf0b4ca854bb9c05698
2024-12-04T15:44:54,791 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-04T15:44:54,791 INFO  [RS:0;a21b6491b371:39841 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-04T15:44:54,791 INFO  [RS:0;a21b6491b371:39841 {}] regionserver.HRegionServer(3091): Received CLOSE for cee5398235ad5e94b1f3bbb5f1071cfe
2024-12-04T15:44:54,792 INFO  [RS:0;a21b6491b371:39841 {}] regionserver.HRegionServer(959): stopping server a21b6491b371,39841,1733327092119
2024-12-04T15:44:54,792 INFO  [RS:0;a21b6491b371:39841 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-04T15:44:54,792 INFO  [RS:0;a21b6491b371:39841 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;a21b6491b371:39841.
2024-12-04T15:44:54,792 INFO  [RS_CLOSE_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1733327093098,,1733327093101.cee5398235ad5e94b1f3bbb5f1071cfe. during a graceful stop, and cache persistence is on, so setting evict on close to false. 
2024-12-04T15:44:54,792 DEBUG [RS:0;a21b6491b371:39841 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-04T15:44:54,792 DEBUG [RS_CLOSE_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing cee5398235ad5e94b1f3bbb5f1071cfe, disabling compactions & flushes
2024-12-04T15:44:54,792 DEBUG [RS:0;a21b6491b371:39841 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-04T15:44:54,792 INFO  [RS_CLOSE_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733327093098,,1733327093101.cee5398235ad5e94b1f3bbb5f1071cfe.
2024-12-04T15:44:54,792 DEBUG [RS_CLOSE_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733327093098,,1733327093101.cee5398235ad5e94b1f3bbb5f1071cfe.
2024-12-04T15:44:54,792 DEBUG [RS_CLOSE_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733327093098,,1733327093101.cee5398235ad5e94b1f3bbb5f1071cfe. after waiting 0 ms
2024-12-04T15:44:54,792 INFO  [RS:0;a21b6491b371:39841 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-04T15:44:54,792 DEBUG [RS_CLOSE_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733327093098,,1733327093101.cee5398235ad5e94b1f3bbb5f1071cfe.
2024-12-04T15:44:54,792 INFO  [RS:0;a21b6491b371:39841 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-04T15:44:54,792 INFO  [RS:0;a21b6491b371:39841 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-04T15:44:54,792 INFO  [RS:0;a21b6491b371:39841 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-04T15:44:54,792 INFO  [RS:0;a21b6491b371:39841 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-12-04T15:44:54,793 DEBUG [RS:0;a21b6491b371:39841 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, cee5398235ad5e94b1f3bbb5f1071cfe=testBlockEvictionOnGracefulStop1733327093098,,1733327093101.cee5398235ad5e94b1f3bbb5f1071cfe.}
2024-12-04T15:44:54,793 DEBUG [RS:0;a21b6491b371:39841 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, cee5398235ad5e94b1f3bbb5f1071cfe
2024-12-04T15:44:54,793 INFO  [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 
2024-12-04T15:44:54,793 DEBUG [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-04T15:44:54,793 INFO  [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-04T15:44:54,793 DEBUG [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-04T15:44:54,793 DEBUG [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-04T15:44:54,793 DEBUG [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-04T15:44:54,793 INFO  [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB
2024-12-04T15:44:54,798 DEBUG [RS_CLOSE_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/default/testBlockEvictionOnGracefulStop1733327093098/cee5398235ad5e94b1f3bbb5f1071cfe/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-12-04T15:44:54,798 INFO  [RS_CLOSE_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733327093098,,1733327093101.cee5398235ad5e94b1f3bbb5f1071cfe.
2024-12-04T15:44:54,798 DEBUG [RS_CLOSE_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for cee5398235ad5e94b1f3bbb5f1071cfe:
Waiting for close lock at 1733327094792Running coprocessor pre-close hooks at 1733327094792Disabling compacts and flushes for region at 1733327094792Disabling writes for close at 1733327094792Writing region close event to WAL at 1733327094793 (+1 ms)Running coprocessor post-close hooks at 1733327094798 (+5 ms)Closed at 1733327094798
2024-12-04T15:44:54,798 DEBUG [RS_CLOSE_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1733327093098,,1733327093101.cee5398235ad5e94b1f3bbb5f1071cfe.
2024-12-04T15:44:54,803 DEBUG [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 894dec8ada2147278c74effb2bcef058
2024-12-04T15:44:54,817 DEBUG [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/.tmp/info/894dec8ada2147278c74effb2bcef058 is 209, key is testBlockEvictionOnGracefulStop1733327093098,,1733327093101.cee5398235ad5e94b1f3bbb5f1071cfe./info:regioninfo/1733327093468/Put/seqid=0
2024-12-04T15:44:54,818 INFO  [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/.tmp/info/894dec8ada2147278c74effb2bcef058
2024-12-04T15:44:54,826 DEBUG [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f23add35bb664e8c8ffd31ceff59860f
2024-12-04T15:44:54,838 DEBUG [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/.tmp/ns/f23add35bb664e8c8ffd31ceff59860f is 43, key is default/ns:d/1733327093063/Put/seqid=0
2024-12-04T15:44:54,839 INFO  [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/.tmp/ns/f23add35bb664e8c8ffd31ceff59860f
2024-12-04T15:44:54,841 INFO  [regionserver/a21b6491b371:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-04T15:44:54,849 DEBUG [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 362037b9e197444ab19a3bb983bd0117
2024-12-04T15:44:54,863 DEBUG [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/.tmp/table/362037b9e197444ab19a3bb983bd0117 is 80, key is testBlockEvictionOnGracefulStop1733327093098/table:state/1733327093477/Put/seqid=0
2024-12-04T15:44:54,864 INFO  [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/.tmp/table/362037b9e197444ab19a3bb983bd0117
2024-12-04T15:44:54,867 DEBUG [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/.tmp/info/894dec8ada2147278c74effb2bcef058 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/info/894dec8ada2147278c74effb2bcef058
2024-12-04T15:44:54,868 DEBUG [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/info/894dec8ada2147278c74effb2bcef058, delay=903 ms
2024-12-04T15:44:54,869 INFO  [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/info/894dec8ada2147278c74effb2bcef058, entries=10, sequenceid=11, filesize=7.0 K
2024-12-04T15:44:54,870 DEBUG [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/.tmp/ns/f23add35bb664e8c8ffd31ceff59860f as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/ns/f23add35bb664e8c8ffd31ceff59860f
2024-12-04T15:44:54,871 DEBUG [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/ns/f23add35bb664e8c8ffd31ceff59860f, delay=929 ms
2024-12-04T15:44:54,872 INFO  [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/ns/f23add35bb664e8c8ffd31ceff59860f, entries=2, sequenceid=11, filesize=5.0 K
2024-12-04T15:44:54,872 DEBUG [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/.tmp/table/362037b9e197444ab19a3bb983bd0117 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/table/362037b9e197444ab19a3bb983bd0117
2024-12-04T15:44:54,873 DEBUG [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/table/362037b9e197444ab19a3bb983bd0117, delay=929 ms
2024-12-04T15:44:54,874 INFO  [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/table/362037b9e197444ab19a3bb983bd0117, entries=2, sequenceid=11, filesize=5.3 K
2024-12-04T15:44:54,875 INFO  [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 81ms, sequenceid=11, compaction requested=false
2024-12-04T15:44:54,875 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/info/894dec8ada2147278c74effb2bcef058
2024-12-04T15:44:54,876 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/ns/f23add35bb664e8c8ffd31ceff59860f
2024-12-04T15:44:54,876 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/table/362037b9e197444ab19a3bb983bd0117
2024-12-04T15:44:54,881 DEBUG [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-12-04T15:44:54,881 DEBUG [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-04T15:44:54,881 INFO  [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-04T15:44:54,881 DEBUG [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733327094793Running coprocessor pre-close hooks at 1733327094793Disabling compacts and flushes for region at 1733327094793Disabling writes for close at 1733327094793Obtaining lock to block concurrent updates at 1733327094793Preparing flush snapshotting stores in 1588230740 at 1733327094793Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1733327094794 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733327094794Flushing 1588230740/info: creating writer at 1733327094794Flushing 1588230740/info: appending metadata at 1733327094816 (+22 ms)Flushing 1588230740/info: closing flushed file at 1733327094816Flushing 1588230740/ns: creating writer at 1733327094820 (+4 ms)Flushing 1588230740/ns: appending metadata at 1733327094838 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1733327094838Flushing 1588230740/table: creating writer at 1733327094842 (+4 ms)Flushing 1588230740/table: appending metadata at 1733327094863 (+21 ms)Flushing 1588230740/table: closing flushed file at 1733327094863Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14606810: reopening flushed file at 1733327094867 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51164603: reopening flushed file at 1733327094869 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3535ff24: reopening flushed file at 1733327094872 (+3 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 81ms, sequenceid=11, compaction requested=false at 1733327094875 (+3 ms)Writing region close event to WAL at 1733327094877 (+2 ms)Running coprocessor post-close hooks at 1733327094881 (+4 ms)Closed at 1733327094881
2024-12-04T15:44:54,881 DEBUG [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-04T15:44:54,993 INFO  [RS:0;a21b6491b371:39841 {}] regionserver.HRegionServer(976): stopping server a21b6491b371,39841,1733327092119; all regions closed.
2024-12-04T15:44:54,995 DEBUG [RS:0;a21b6491b371:39841 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/oldWALs
2024-12-04T15:44:54,995 INFO  [RS:0;a21b6491b371:39841 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a21b6491b371%2C39841%2C1733327092119.meta:.meta(num 1733327093019)
2024-12-04T15:44:54,997 DEBUG [RS:0;a21b6491b371:39841 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/oldWALs
2024-12-04T15:44:54,997 INFO  [RS:0;a21b6491b371:39841 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a21b6491b371%2C39841%2C1733327092119:(num 1733327092677)
2024-12-04T15:44:54,997 DEBUG [RS:0;a21b6491b371:39841 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-04T15:44:54,997 INFO  [RS:0;a21b6491b371:39841 {}] regionserver.LeaseManager(133): Closed leases
2024-12-04T15:44:54,997 INFO  [RS:0;a21b6491b371:39841 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-04T15:44:54,997 INFO  [RS:0;a21b6491b371:39841 {}] hbase.ChoreService(370): Chore service for: regionserver/a21b6491b371:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-12-04T15:44:54,997 INFO  [RS:0;a21b6491b371:39841 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-04T15:44:54,997 INFO  [regionserver/a21b6491b371:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-04T15:44:54,997 INFO  [RS:0;a21b6491b371:39841 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39841
2024-12-04T15:44:55,001 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39841-0x1017e2663e10001, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a21b6491b371,39841,1733327092119
2024-12-04T15:44:55,001 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44449-0x1017e2663e10000, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-04T15:44:55,002 INFO  [RS:0;a21b6491b371:39841 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-04T15:44:55,003 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a21b6491b371,39841,1733327092119]
2024-12-04T15:44:55,004 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a21b6491b371,39841,1733327092119 already deleted, retry=false
2024-12-04T15:44:55,005 INFO  [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of a21b6491b371,39841,1733327092119 on a21b6491b371,44449,1733327092066
2024-12-04T15:44:55,009 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure a21b6491b371,39841,1733327092119, splitWal=true, meta=true
2024-12-04T15:44:55,011 INFO  [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for a21b6491b371,39841,1733327092119 (carryingMeta=true) a21b6491b371,39841,1733327092119/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@14773029[Write locks = 1, Read locks = 0], oldState=ONLINE.
2024-12-04T15:44:55,011 INFO  [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure a21b6491b371,39841,1733327092119, splitWal=true, meta=true
2024-12-04T15:44:55,012 INFO  [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure a21b6491b371,39841,1733327092119, splitWal=true, meta=true, isMeta: true
2024-12-04T15:44:55,013 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/WALs/a21b6491b371,39841,1733327092119-splitting
2024-12-04T15:44:55,013 INFO  [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/WALs/a21b6491b371,39841,1733327092119-splitting dir is empty, no logs to split.
2024-12-04T15:44:55,013 INFO  [PEWorker-2 {}] master.SplitWALManager(105): a21b6491b371,39841,1733327092119 WAL count=0, meta=true
2024-12-04T15:44:55,015 INFO  [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/WALs/a21b6491b371,39841,1733327092119-splitting dir is empty, no logs to split.
2024-12-04T15:44:55,015 INFO  [PEWorker-2 {}] master.SplitWALManager(105): a21b6491b371,39841,1733327092119 WAL count=0, meta=true
2024-12-04T15:44:55,015 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if a21b6491b371,39841,1733327092119 WAL splitting is done? wals=0, meta=true
2024-12-04T15:44:55,017 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-12-04T15:44:55,017 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-04T15:44:55,018 INFO  [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false
2024-12-04T15:44:55,103 INFO  [RS:0;a21b6491b371:39841 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-04T15:44:55,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39841-0x1017e2663e10001, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-04T15:44:55,103 INFO  [RS:0;a21b6491b371:39841 {}] regionserver.HRegionServer(1031): Exiting; stopping=a21b6491b371,39841,1733327092119; zookeeper connection closed.
2024-12-04T15:44:55,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39841-0x1017e2663e10001, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-04T15:44:55,104 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2349bb53 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2349bb53
2024-12-04T15:44:55,168 DEBUG [a21b6491b371:44449 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-12-04T15:44:55,169 INFO  [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a21b6491b371,34119,1733327092157
2024-12-04T15:44:55,170 INFO  [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a21b6491b371,34119,1733327092157, state=OPENING
2024-12-04T15:44:55,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44449-0x1017e2663e10000, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-04T15:44:55,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34119-0x1017e2663e10002, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-04T15:44:55,172 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-04T15:44:55,172 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-04T15:44:55,172 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-12-04T15:44:55,173 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a21b6491b371,34119,1733327092157}]
2024-12-04T15:44:55,293 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/a21b6491b371:0 server-side Connection retries=45
2024-12-04T15:44:55,293 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-04T15:44:55,293 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-12-04T15:44:55,293 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-12-04T15:44:55,293 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-12-04T15:44:55,293 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-12-04T15:44:55,293 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-12-04T15:44:55,293 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-12-04T15:44:55,294 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38071
2024-12-04T15:44:55,296 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38071 connecting to ZooKeeper ensemble=127.0.0.1:58881
2024-12-04T15:44:55,296 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-04T15:44:55,296 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-04T15:44:55,299 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:380710x0, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-12-04T15:44:55,299 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38071-0x1017e2663e10003 connected
2024-12-04T15:44:55,299 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:38071-0x1017e2663e10003, quorum=127.0.0.1:58881, baseZNode=/hbase Set watcher on existing znode=/hbase/running
2024-12-04T15:44:55,299 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT
2024-12-04T15:44:55,300 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-12-04T15:44:55,301 INFO  [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/de2c1fb8-a351-04c3-cfcb-b73746afa260/bucket.cache
2024-12-04T15:44:55,301 INFO  [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false
2024-12-04T15:44:55,301 INFO  [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread
2024-12-04T15:44:55,301 INFO  [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100
2024-12-04T15:44:55,301 INFO  [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/de2c1fb8-a351-04c3-cfcb-b73746afa260/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/de2c1fb8-a351-04c3-cfcb-b73746afa260/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator
2024-12-04T15:44:55,301 INFO  [Thread-206 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file
2024-12-04T15:44:55,301 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-12-04T15:44:55,302 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:38071-0x1017e2663e10003, quorum=127.0.0.1:58881, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-12-04T15:44:55,302 INFO  [Thread-206 {}] bucket.BucketCache(1552): Reading new chunked format of persistence.
2024-12-04T15:44:55,303 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38071-0x1017e2663e10003, quorum=127.0.0.1:58881, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-12-04T15:44:55,304 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38071
2024-12-04T15:44:55,304 INFO  [Thread-206 {}] bucket.BucketCache(1705): Retrieved 1 of chunks with blockCount = 1.
2024-12-04T15:44:55,304 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38071
2024-12-04T15:44:55,304 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38071
2024-12-04T15:44:55,306 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38071
2024-12-04T15:44:55,307 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38071
2024-12-04T15:44:55,307 DEBUG [Thread-206 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list
2024-12-04T15:44:55,308 INFO  [Thread-206 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size))
2024-12-04T15:44:55,308 INFO  [Thread-206 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 1
2024-12-04T15:44:55,308 INFO  [Thread-206 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/de2c1fb8-a351-04c3-cfcb-b73746afa260/bucket.persistence is complete.
2024-12-04T15:44:55,308 INFO  [RS:2;a21b6491b371:38071 {}] regionserver.HRegionServer(746): ClusterId : 9b485e3f-9db5-4ebe-a775-c51db92356e7
2024-12-04T15:44:55,308 DEBUG [RS:2;a21b6491b371:38071 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-12-04T15:44:55,310 DEBUG [RS:2;a21b6491b371:38071 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-12-04T15:44:55,310 DEBUG [RS:2;a21b6491b371:38071 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-12-04T15:44:55,312 DEBUG [RS:2;a21b6491b371:38071 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-12-04T15:44:55,313 DEBUG [RS:2;a21b6491b371:38071 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bb74a6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a21b6491b371/172.17.0.2:0
2024-12-04T15:44:55,325 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-04T15:44:55,326 DEBUG [RS:2;a21b6491b371:38071 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;a21b6491b371:38071
2024-12-04T15:44:55,326 INFO  [RS:2;a21b6491b371:38071 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-12-04T15:44:55,326 INFO  [RS:2;a21b6491b371:38071 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-12-04T15:44:55,326 DEBUG [RS:2;a21b6491b371:38071 {}] regionserver.HRegionServer(832): About to register with Master.
2024-12-04T15:44:55,327 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55415, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-04T15:44:55,327 INFO  [RS:2;a21b6491b371:38071 {}] regionserver.HRegionServer(2659): reportForDuty to master=a21b6491b371,44449,1733327092066 with port=38071, startcode=1733327095293
2024-12-04T15:44:55,327 DEBUG [RS:2;a21b6491b371:38071 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-12-04T15:44:55,329 INFO  [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-12-04T15:44:55,329 INFO  [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider
2024-12-04T15:44:55,329 INFO  [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35025, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService
2024-12-04T15:44:55,329 INFO  [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta
2024-12-04T15:44:55,330 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44449 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a21b6491b371,38071,1733327095293
2024-12-04T15:44:55,330 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44449 {}] master.ServerManager(517): Registering regionserver=a21b6491b371,38071,1733327095293
2024-12-04T15:44:55,330 INFO  [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=a21b6491b371%2C34119%2C1733327092157.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/WALs/a21b6491b371,34119,1733327092157, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/oldWALs, maxLogs=55
2024-12-04T15:44:55,331 DEBUG [RS:2;a21b6491b371:38071 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621
2024-12-04T15:44:55,331 DEBUG [RS:2;a21b6491b371:38071 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/
2024-12-04T15:44:55,331 DEBUG [RS:2;a21b6491b371:38071 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-12-04T15:44:55,334 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44449-0x1017e2663e10000, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-04T15:44:55,335 DEBUG [RS:2;a21b6491b371:38071 {}] zookeeper.ZKUtil(111): regionserver:38071-0x1017e2663e10003, quorum=127.0.0.1:58881, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a21b6491b371,38071,1733327095293
2024-12-04T15:44:55,335 WARN  [RS:2;a21b6491b371:38071 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-12-04T15:44:55,335 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a21b6491b371,38071,1733327095293]
2024-12-04T15:44:55,335 INFO  [RS:2;a21b6491b371:38071 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider
2024-12-04T15:44:55,335 DEBUG [RS:2;a21b6491b371:38071 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/WALs/a21b6491b371,38071,1733327095293
2024-12-04T15:44:55,341 INFO  [RS:2;a21b6491b371:38071 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-12-04T15:44:55,343 INFO  [RS:2;a21b6491b371:38071 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-12-04T15:44:55,344 INFO  [RS:2;a21b6491b371:38071 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-12-04T15:44:55,344 INFO  [RS:2;a21b6491b371:38071 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:55,344 INFO  [RS:2;a21b6491b371:38071 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-12-04T15:44:55,345 INFO  [RS:2;a21b6491b371:38071 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-12-04T15:44:55,345 INFO  [RS:2;a21b6491b371:38071 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:55,345 DEBUG [RS:2;a21b6491b371:38071 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:55,345 DEBUG [RS:2;a21b6491b371:38071 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:55,345 DEBUG [RS:2;a21b6491b371:38071 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:55,345 DEBUG [RS:2;a21b6491b371:38071 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:55,346 DEBUG [RS:2;a21b6491b371:38071 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:55,346 DEBUG [RS:2;a21b6491b371:38071 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a21b6491b371:0, corePoolSize=2, maxPoolSize=2
2024-12-04T15:44:55,346 DEBUG [RS:2;a21b6491b371:38071 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:55,346 DEBUG [RS:2;a21b6491b371:38071 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:55,346 DEBUG [RS:2;a21b6491b371:38071 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:55,346 DEBUG [RS:2;a21b6491b371:38071 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:55,346 DEBUG [RS:2;a21b6491b371:38071 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:55,346 DEBUG [RS:2;a21b6491b371:38071 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a21b6491b371:0, corePoolSize=1, maxPoolSize=1
2024-12-04T15:44:55,346 DEBUG [RS:2;a21b6491b371:38071 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a21b6491b371:0, corePoolSize=3, maxPoolSize=3
2024-12-04T15:44:55,346 DEBUG [RS:2;a21b6491b371:38071 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a21b6491b371:0, corePoolSize=3, maxPoolSize=3
2024-12-04T15:44:55,346 INFO  [RS:2;a21b6491b371:38071 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:55,347 INFO  [RS:2;a21b6491b371:38071 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:55,347 INFO  [RS:2;a21b6491b371:38071 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:55,347 INFO  [RS:2;a21b6491b371:38071 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:55,347 INFO  [RS:2;a21b6491b371:38071 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:55,347 INFO  [RS:2;a21b6491b371:38071 {}] hbase.ChoreService(168): Chore ScheduledChore name=a21b6491b371,38071,1733327095293-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-12-04T15:44:55,353 INFO  [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/WALs/a21b6491b371,34119,1733327092157/a21b6491b371%2C34119%2C1733327092157.meta.1733327095331.meta
2024-12-04T15:44:55,353 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: []
2024-12-04T15:44:55,354 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-12-04T15:44:55,354 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-12-04T15:44:55,354 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-12-04T15:44:55,355 INFO  [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-12-04T15:44:55,355 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-12-04T15:44:55,355 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-04T15:44:55,355 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-12-04T15:44:55,355 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-12-04T15:44:55,356 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 
2024-12-04T15:44:55,357 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-12-04T15:44:55,357 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-04T15:44:55,358 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/info/894dec8ada2147278c74effb2bcef058, delay=932 ms
2024-12-04T15:44:55,359 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/info/894dec8ada2147278c74effb2bcef058
2024-12-04T15:44:55,360 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-04T15:44:55,360 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 
2024-12-04T15:44:55,360 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-12-04T15:44:55,360 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-04T15:44:55,362 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/ns/f23add35bb664e8c8ffd31ceff59860f, delay=951 ms
2024-12-04T15:44:55,363 INFO  [RS:2;a21b6491b371:38071 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-12-04T15:44:55,363 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/ns/f23add35bb664e8c8ffd31ceff59860f
2024-12-04T15:44:55,363 INFO  [RS:2;a21b6491b371:38071 {}] hbase.ChoreService(168): Chore ScheduledChore name=a21b6491b371,38071,1733327095293-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:55,363 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-04T15:44:55,363 INFO  [RS:2;a21b6491b371:38071 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:55,363 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 
2024-12-04T15:44:55,363 INFO  [RS:2;a21b6491b371:38071 {}] regionserver.Replication(171): a21b6491b371,38071,1733327095293 started
2024-12-04T15:44:55,364 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-12-04T15:44:55,364 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-04T15:44:55,364 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-04T15:44:55,364 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 
2024-12-04T15:44:55,365 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-12-04T15:44:55,365 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-04T15:44:55,366 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/table/362037b9e197444ab19a3bb983bd0117, delay=984 ms
2024-12-04T15:44:55,367 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/table/362037b9e197444ab19a3bb983bd0117
2024-12-04T15:44:55,367 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-12-04T15:44:55,368 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-12-04T15:44:55,368 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740
2024-12-04T15:44:55,369 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740
2024-12-04T15:44:55,369 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-12-04T15:44:55,369 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-12-04T15:44:55,370 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-12-04T15:44:55,370 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-12-04T15:44:55,371 INFO  [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65759397, jitterRate=-0.020108625292778015}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-12-04T15:44:55,371 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-12-04T15:44:55,372 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1733327095355Writing region info on filesystem at 1733327095355Initializing all the Stores at 1733327095356 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733327095356Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733327095356Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733327095356Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733327095356Cleaning up temporary data from old regions at 1733327095369 (+13 ms)Running coprocessor post-open hooks at 1733327095371 (+2 ms)Region opened successfully at 1733327095372 (+1 ms)
2024-12-04T15:44:55,373 INFO  [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1733327095325
2024-12-04T15:44:55,375 DEBUG [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-12-04T15:44:55,375 INFO  [RS_OPEN_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-12-04T15:44:55,376 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=a21b6491b371,34119,1733327092157
2024-12-04T15:44:55,377 INFO  [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a21b6491b371,34119,1733327092157, state=OPEN
2024-12-04T15:44:55,378 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34119-0x1017e2663e10002, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-04T15:44:55,378 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38071-0x1017e2663e10003, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-04T15:44:55,378 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44449-0x1017e2663e10000, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-12-04T15:44:55,379 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a21b6491b371,34119,1733327092157
2024-12-04T15:44:55,379 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-04T15:44:55,379 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-04T15:44:55,379 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-12-04T15:44:55,380 INFO  [RS:2;a21b6491b371:38071 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-12-04T15:44:55,380 INFO  [RS:2;a21b6491b371:38071 {}] regionserver.HRegionServer(1482): Serving as a21b6491b371,38071,1733327095293, RpcServer on a21b6491b371/172.17.0.2:38071, sessionid=0x1017e2663e10003
2024-12-04T15:44:55,380 DEBUG [RS:2;a21b6491b371:38071 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-12-04T15:44:55,381 DEBUG [RS:2;a21b6491b371:38071 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a21b6491b371,38071,1733327095293
2024-12-04T15:44:55,381 DEBUG [RS:2;a21b6491b371:38071 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a21b6491b371,38071,1733327095293'
2024-12-04T15:44:55,381 DEBUG [RS:2;a21b6491b371:38071 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-12-04T15:44:55,381 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8
2024-12-04T15:44:55,381 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a21b6491b371,34119,1733327092157 in 206 msec
2024-12-04T15:44:55,381 DEBUG [RS:2;a21b6491b371:38071 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-12-04T15:44:55,382 DEBUG [RS:2;a21b6491b371:38071 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-12-04T15:44:55,382 DEBUG [RS:2;a21b6491b371:38071 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-12-04T15:44:55,382 DEBUG [RS:2;a21b6491b371:38071 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a21b6491b371,38071,1733327095293
2024-12-04T15:44:55,382 DEBUG [RS:2;a21b6491b371:38071 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a21b6491b371,38071,1733327095293'
2024-12-04T15:44:55,382 DEBUG [RS:2;a21b6491b371:38071 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-12-04T15:44:55,382 DEBUG [RS:2;a21b6491b371:38071 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-12-04T15:44:55,383 DEBUG [RS:2;a21b6491b371:38071 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-12-04T15:44:55,383 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7
2024-12-04T15:44:55,383 INFO  [RS:2;a21b6491b371:38071 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-12-04T15:44:55,383 INFO  [RS:2;a21b6491b371:38071 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-12-04T15:44:55,383 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 365 msec
2024-12-04T15:44:55,383 INFO  [PEWorker-3 {}] procedure.ServerCrashProcedure(207): a21b6491b371,39841,1733327092119 had 2 regions
2024-12-04T15:44:55,384 INFO  [PEWorker-3 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure a21b6491b371,39841,1733327092119, splitWal=true, meta=true, isMeta: false
2024-12-04T15:44:55,385 INFO  [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/WALs/a21b6491b371,39841,1733327092119-splitting dir is empty, no logs to split.
2024-12-04T15:44:55,385 INFO  [PEWorker-3 {}] master.SplitWALManager(105): a21b6491b371,39841,1733327092119 WAL count=0, meta=false
2024-12-04T15:44:55,386 INFO  [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/WALs/a21b6491b371,39841,1733327092119-splitting dir is empty, no logs to split.
2024-12-04T15:44:55,386 INFO  [PEWorker-3 {}] master.SplitWALManager(105): a21b6491b371,39841,1733327092119 WAL count=0, meta=false
2024-12-04T15:44:55,386 DEBUG [PEWorker-3 {}] procedure.ServerCrashProcedure(329): Check if a21b6491b371,39841,1733327092119 WAL splitting is done? wals=0, meta=false
2024-12-04T15:44:55,387 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733327093098, region=cee5398235ad5e94b1f3bbb5f1071cfe, ASSIGN}]
2024-12-04T15:44:55,388 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733327093098, region=cee5398235ad5e94b1f3bbb5f1071cfe, ASSIGN
2024-12-04T15:44:55,389 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733327093098, region=cee5398235ad5e94b1f3bbb5f1071cfe, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false
2024-12-04T15:44:55,484 INFO  [RS:2;a21b6491b371:38071 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName
2024-12-04T15:44:55,484 INFO  [RS:2;a21b6491b371:38071 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=a21b6491b371%2C38071%2C1733327095293, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/WALs/a21b6491b371,38071,1733327095293, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/oldWALs, maxLogs=55
2024-12-04T15:44:55,504 INFO  [RS:2;a21b6491b371:38071 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/WALs/a21b6491b371,38071,1733327095293/a21b6491b371%2C38071%2C1733327095293.1733327095485
2024-12-04T15:44:55,504 DEBUG [RS:2;a21b6491b371:38071 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: []
2024-12-04T15:44:55,539 DEBUG [a21b6491b371:44449 {}] balancer.BalancerClusterState(204): Hosts are {a21b6491b371=0} racks are {/default-rack=0}
2024-12-04T15:44:55,539 DEBUG [a21b6491b371:44449 {}] balancer.BalancerClusterState(303): server 0 has 0 regions
2024-12-04T15:44:55,539 DEBUG [a21b6491b371:44449 {}] balancer.BalancerClusterState(303): server 1 has 0 regions
2024-12-04T15:44:55,539 DEBUG [a21b6491b371:44449 {}] balancer.BalancerClusterState(310): server 0 is on host 0
2024-12-04T15:44:55,539 DEBUG [a21b6491b371:44449 {}] balancer.BalancerClusterState(310): server 1 is on host 0
2024-12-04T15:44:55,539 INFO  [a21b6491b371:44449 {}] balancer.BalancerClusterState(321): server 0 is on rack 0
2024-12-04T15:44:55,539 INFO  [a21b6491b371:44449 {}] balancer.BalancerClusterState(321): server 1 is on rack 0
2024-12-04T15:44:55,539 DEBUG [a21b6491b371:44449 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1
2024-12-04T15:44:55,540 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=cee5398235ad5e94b1f3bbb5f1071cfe, regionState=OPENING, regionLocation=a21b6491b371,38071,1733327095293
2024-12-04T15:44:55,541 WARN  [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server a21b6491b371:39841
org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: a21b6491b371/172.17.0.2:39841
Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused
	at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?]
	at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-12-04T15:44:55,543 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=a21b6491b371,39841,1733327092119, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=a21b6491b371,39841,1733327092119, seqNum=-1, error=java.net.ConnectException: Call to address=a21b6491b371:39841 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: a21b6491b371/172.17.0.2:39841
2024-12-04T15:44:55,544 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=a21b6491b371,39841,1733327092119, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused
2024-12-04T15:44:55,544 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=a21b6491b371,39841,1733327092119, seqNum=-1 from cache
2024-12-04T15:44:55,545 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.FailedServers(52): Added failed server with address a21b6491b371:39841 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: a21b6491b371/172.17.0.2:39841
2024-12-04T15:44:55,652 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-12-04T15:44:55,652 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a21b6491b371,34119,1733327092157, seqNum=-1]
2024-12-04T15:44:55,652 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-12-04T15:44:55,654 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45583, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-12-04T15:44:55,656 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733327093098, region=cee5398235ad5e94b1f3bbb5f1071cfe, ASSIGN because future has completed
2024-12-04T15:44:55,656 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure cee5398235ad5e94b1f3bbb5f1071cfe, server=a21b6491b371,38071,1733327095293}]
2024-12-04T15:44:55,809 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-12-04T15:44:55,811 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-9-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60273, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-12-04T15:44:55,813 INFO  [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1733327093098,,1733327093101.cee5398235ad5e94b1f3bbb5f1071cfe.
2024-12-04T15:44:55,813 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7752): Opening region: {ENCODED => cee5398235ad5e94b1f3bbb5f1071cfe, NAME => 'testBlockEvictionOnGracefulStop1733327093098,,1733327093101.cee5398235ad5e94b1f3bbb5f1071cfe.', STARTKEY => '', ENDKEY => ''}
2024-12-04T15:44:55,814 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1733327093098 cee5398235ad5e94b1f3bbb5f1071cfe
2024-12-04T15:44:55,814 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733327093098,,1733327093101.cee5398235ad5e94b1f3bbb5f1071cfe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-12-04T15:44:55,814 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7794): checking encryption for cee5398235ad5e94b1f3bbb5f1071cfe
2024-12-04T15:44:55,814 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7797): checking classloading for cee5398235ad5e94b1f3bbb5f1071cfe
2024-12-04T15:44:55,815 INFO  [StoreOpener-cee5398235ad5e94b1f3bbb5f1071cfe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region cee5398235ad5e94b1f3bbb5f1071cfe 
2024-12-04T15:44:55,816 INFO  [StoreOpener-cee5398235ad5e94b1f3bbb5f1071cfe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cee5398235ad5e94b1f3bbb5f1071cfe columnFamilyName family
2024-12-04T15:44:55,816 DEBUG [StoreOpener-cee5398235ad5e94b1f3bbb5f1071cfe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-12-04T15:44:55,818 DEBUG [StoreFileOpener-cee5398235ad5e94b1f3bbb5f1071cfe-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/default/testBlockEvictionOnGracefulStop1733327093098/cee5398235ad5e94b1f3bbb5f1071cfe/family/b36d40b7dadc4cf0b4ca854bb9c05698, delay=945 ms
2024-12-04T15:44:55,819 DEBUG [StoreOpener-cee5398235ad5e94b1f3bbb5f1071cfe-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/default/testBlockEvictionOnGracefulStop1733327093098/cee5398235ad5e94b1f3bbb5f1071cfe/family/b36d40b7dadc4cf0b4ca854bb9c05698
2024-12-04T15:44:55,819 INFO  [StoreOpener-cee5398235ad5e94b1f3bbb5f1071cfe-1 {}] regionserver.HStore(327): Store=cee5398235ad5e94b1f3bbb5f1071cfe/family,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-12-04T15:44:55,819 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1038): replaying wal for cee5398235ad5e94b1f3bbb5f1071cfe
2024-12-04T15:44:55,820 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/default/testBlockEvictionOnGracefulStop1733327093098/cee5398235ad5e94b1f3bbb5f1071cfe
2024-12-04T15:44:55,820 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/default/testBlockEvictionOnGracefulStop1733327093098/cee5398235ad5e94b1f3bbb5f1071cfe
2024-12-04T15:44:55,821 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1048): stopping wal replay for cee5398235ad5e94b1f3bbb5f1071cfe
2024-12-04T15:44:55,821 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1060): Cleaning up temporary data for cee5398235ad5e94b1f3bbb5f1071cfe
2024-12-04T15:44:55,822 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): writing seq id for cee5398235ad5e94b1f3bbb5f1071cfe
2024-12-04T15:44:55,822 INFO  [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1114): Opened cee5398235ad5e94b1f3bbb5f1071cfe; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58774000, jitterRate=-0.12419915199279785}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-12-04T15:44:55,822 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cee5398235ad5e94b1f3bbb5f1071cfe
2024-12-04T15:44:55,823 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1006): Region open journal for cee5398235ad5e94b1f3bbb5f1071cfe:
Running coprocessor pre-open hook at 1733327095814Writing region info on filesystem at 1733327095814Initializing all the Stores at 1733327095814Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733327095814Cleaning up temporary data from old regions at 1733327095821 (+7 ms)Running coprocessor post-open hooks at 1733327095822 (+1 ms)Region opened successfully at 1733327095823 (+1 ms)
2024-12-04T15:44:55,824 INFO  [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1733327093098,,1733327093101.cee5398235ad5e94b1f3bbb5f1071cfe., pid=11, masterSystemTime=1733327095808
2024-12-04T15:44:55,827 DEBUG [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1733327093098,,1733327093101.cee5398235ad5e94b1f3bbb5f1071cfe.
2024-12-04T15:44:55,827 INFO  [RS_OPEN_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1733327093098,,1733327093101.cee5398235ad5e94b1f3bbb5f1071cfe.
2024-12-04T15:44:55,827 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=cee5398235ad5e94b1f3bbb5f1071cfe, regionState=OPEN, openSeqNum=10, regionLocation=a21b6491b371,38071,1733327095293
2024-12-04T15:44:55,830 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure cee5398235ad5e94b1f3bbb5f1071cfe, server=a21b6491b371,38071,1733327095293 because future has completed
2024-12-04T15:44:55,832 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10
2024-12-04T15:44:55,832 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure cee5398235ad5e94b1f3bbb5f1071cfe, server=a21b6491b371,38071,1733327095293 in 175 msec
2024-12-04T15:44:55,834 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7
2024-12-04T15:44:55,834 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733327093098, region=cee5398235ad5e94b1f3bbb5f1071cfe, ASSIGN in 445 msec
2024-12-04T15:44:55,834 INFO  [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server a21b6491b371,39841,1733327092119 after splitting done
2024-12-04T15:44:55,836 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure a21b6491b371,39841,1733327092119, splitWal=true, meta=true in 829 msec
2024-12-04T15:44:55,880 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-12-04T15:44:55,881 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-04T15:44:55,881 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-04T15:44:55,881 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-04T15:44:55,881 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-04T15:44:55,881 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-12-04T15:44:55,881 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-12-04T15:44:55,881 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=203969093, stopped=false
2024-12-04T15:44:55,881 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=a21b6491b371,44449,1733327092066
2024-12-04T15:44:55,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38071-0x1017e2663e10003, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-04T15:44:55,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44449-0x1017e2663e10000, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-04T15:44:55,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34119-0x1017e2663e10002, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-12-04T15:44:55,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38071-0x1017e2663e10003, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-04T15:44:55,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44449-0x1017e2663e10000, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-04T15:44:55,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34119-0x1017e2663e10002, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-04T15:44:55,883 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-04T15:44:55,884 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-12-04T15:44:55,884 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44449-0x1017e2663e10000, quorum=127.0.0.1:58881, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-04T15:44:55,884 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-04T15:44:55,884 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-04T15:44:55,884 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38071-0x1017e2663e10003, quorum=127.0.0.1:58881, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-04T15:44:55,884 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a21b6491b371,34119,1733327092157' *****
2024-12-04T15:44:55,884 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-04T15:44:55,885 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a21b6491b371,38071,1733327095293' *****
2024-12-04T15:44:55,885 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-12-04T15:44:55,885 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34119-0x1017e2663e10002, quorum=127.0.0.1:58881, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-12-04T15:44:55,885 INFO  [RS:1;a21b6491b371:34119 {}] bucket.BucketCache(1738): Disabling cache
2024-12-04T15:44:55,885 INFO  [RS:2;a21b6491b371:38071 {}] bucket.BucketCache(1738): Disabling cache
2024-12-04T15:44:55,885 INFO  [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false
2024-12-04T15:44:55,885 INFO  [RS:1;a21b6491b371:34119 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/de2c1fb8-a351-04c3-cfcb-b73746afa260/bucket.persistence
2024-12-04T15:44:55,885 INFO  [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false
2024-12-04T15:44:55,885 INFO  [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false
2024-12-04T15:44:55,885 INFO  [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false
2024-12-04T15:44:55,885 INFO  [RS:2;a21b6491b371:38071 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/de2c1fb8-a351-04c3-cfcb-b73746afa260/bucket.persistence
2024-12-04T15:44:55,885 INFO  [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false
2024-12-04T15:44:55,885 INFO  [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false
2024-12-04T15:44:55,886 INFO  [RS:1;a21b6491b371:34119 {}] bucket.BucketCache(1768): Shutting down cache persister thread.
2024-12-04T15:44:55,886 INFO  [RS:2;a21b6491b371:38071 {}] bucket.BucketCache(1768): Shutting down cache persister thread.
2024-12-04T15:44:55,886 WARN  [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread.
java.lang.InterruptedException: sleep interrupted
	at java.lang.Thread.sleep(Native Method) ~[?:?]
	at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?]
2024-12-04T15:44:55,886 WARN  [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread.
java.lang.InterruptedException: sleep interrupted
	at java.lang.Thread.sleep(Native Method) ~[?:?]
	at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?]
2024-12-04T15:44:55,896 DEBUG [RS:2;a21b6491b371:38071 {}] bucket.BucketCache(1487): Thread RS:2;a21b6491b371:38071 started persisting bucket cache to file
2024-12-04T15:44:55,896 DEBUG [RS:1;a21b6491b371:34119 {}] bucket.BucketCache(1487): Thread RS:1;a21b6491b371:34119 started persisting bucket cache to file
2024-12-04T15:44:55,896 DEBUG [RS:2;a21b6491b371:38071 {}] bucket.BucketCache(1494): Persist in new chunked persistence format.
2024-12-04T15:44:55,896 DEBUG [RS:1;a21b6491b371:34119 {}] bucket.BucketCache(1494): Persist in new chunked persistence format.
2024-12-04T15:44:55,896 DEBUG [RS:2;a21b6491b371:38071 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000
2024-12-04T15:44:55,896 DEBUG [RS:1;a21b6491b371:34119 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000
2024-12-04T15:44:55,900 DEBUG [RS:1;a21b6491b371:34119 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0
2024-12-04T15:44:55,900 DEBUG [RS:1;a21b6491b371:34119 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733327095896
2024-12-04T15:44:55,900 DEBUG [RS:2;a21b6491b371:38071 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0
2024-12-04T15:44:55,900 DEBUG [RS:1;a21b6491b371:34119 {}] bucket.BucketCache(1509): Thread RS:1;a21b6491b371:34119 finished persisting bucket cache to file, renaming
2024-12-04T15:44:55,900 DEBUG [RS:2;a21b6491b371:38071 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1733327095896
2024-12-04T15:44:55,900 DEBUG [RS:2;a21b6491b371:38071 {}] bucket.BucketCache(1509): Thread RS:2;a21b6491b371:38071 finished persisting bucket cache to file, renaming
2024-12-04T15:44:55,900 WARN  [RS:2;a21b6491b371:38071 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again.
2024-12-04T15:44:55,900 INFO  [RS:2;a21b6491b371:38071 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-04T15:44:55,900 INFO  [RS:1;a21b6491b371:34119 {}] regionserver.HeapMemoryManager(220): Stopping
2024-12-04T15:44:55,900 INFO  [RS:2;a21b6491b371:38071 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-04T15:44:55,900 INFO  [RS:1;a21b6491b371:34119 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-12-04T15:44:55,900 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-04T15:44:55,900 INFO  [RS:2;a21b6491b371:38071 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-04T15:44:55,900 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-12-04T15:44:55,900 INFO  [RS:1;a21b6491b371:34119 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-12-04T15:44:55,900 INFO  [RS:2;a21b6491b371:38071 {}] regionserver.HRegionServer(3091): Received CLOSE for cee5398235ad5e94b1f3bbb5f1071cfe
2024-12-04T15:44:55,900 INFO  [RS:1;a21b6491b371:34119 {}] regionserver.HRegionServer(959): stopping server a21b6491b371,34119,1733327092157
2024-12-04T15:44:55,901 INFO  [RS:1;a21b6491b371:34119 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-04T15:44:55,901 INFO  [RS:1;a21b6491b371:34119 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;a21b6491b371:34119.
2024-12-04T15:44:55,901 DEBUG [RS:1;a21b6491b371:34119 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-04T15:44:55,901 INFO  [RS:2;a21b6491b371:38071 {}] regionserver.HRegionServer(959): stopping server a21b6491b371,38071,1733327095293
2024-12-04T15:44:55,901 DEBUG [RS:1;a21b6491b371:34119 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-04T15:44:55,901 INFO  [RS:2;a21b6491b371:38071 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-04T15:44:55,901 INFO  [RS:1;a21b6491b371:34119 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-04T15:44:55,901 INFO  [RS:2;a21b6491b371:38071 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;a21b6491b371:38071.
2024-12-04T15:44:55,901 INFO  [RS:1;a21b6491b371:34119 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-04T15:44:55,901 INFO  [RS_CLOSE_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1733327093098,,1733327093101.cee5398235ad5e94b1f3bbb5f1071cfe. during a graceful stop, and cache persistence is on, so setting evict on close to false. 
2024-12-04T15:44:55,901 INFO  [RS:1;a21b6491b371:34119 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-04T15:44:55,901 INFO  [RS:1;a21b6491b371:34119 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-12-04T15:44:55,901 DEBUG [RS_CLOSE_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing cee5398235ad5e94b1f3bbb5f1071cfe, disabling compactions & flushes
2024-12-04T15:44:55,901 DEBUG [RS:2;a21b6491b371:38071 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-12-04T15:44:55,901 DEBUG [RS:2;a21b6491b371:38071 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-04T15:44:55,901 INFO  [RS_CLOSE_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733327093098,,1733327093101.cee5398235ad5e94b1f3bbb5f1071cfe.
2024-12-04T15:44:55,901 DEBUG [RS_CLOSE_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733327093098,,1733327093101.cee5398235ad5e94b1f3bbb5f1071cfe.
2024-12-04T15:44:55,901 INFO  [RS:2;a21b6491b371:38071 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-12-04T15:44:55,901 DEBUG [RS_CLOSE_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733327093098,,1733327093101.cee5398235ad5e94b1f3bbb5f1071cfe. after waiting 0 ms
2024-12-04T15:44:55,901 DEBUG [RS:2;a21b6491b371:38071 {}] regionserver.HRegionServer(1325): Online Regions={cee5398235ad5e94b1f3bbb5f1071cfe=testBlockEvictionOnGracefulStop1733327093098,,1733327093101.cee5398235ad5e94b1f3bbb5f1071cfe.}
2024-12-04T15:44:55,901 DEBUG [RS_CLOSE_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733327093098,,1733327093101.cee5398235ad5e94b1f3bbb5f1071cfe.
2024-12-04T15:44:55,901 DEBUG [RS:2;a21b6491b371:38071 {}] regionserver.HRegionServer(1351): Waiting on cee5398235ad5e94b1f3bbb5f1071cfe
2024-12-04T15:44:55,901 INFO  [RS:1;a21b6491b371:34119 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-12-04T15:44:55,902 DEBUG [RS:1;a21b6491b371:34119 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-12-04T15:44:55,902 INFO  [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 
2024-12-04T15:44:55,902 DEBUG [RS:1;a21b6491b371:34119 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-12-04T15:44:55,902 DEBUG [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-12-04T15:44:55,902 INFO  [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-12-04T15:44:55,902 DEBUG [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-12-04T15:44:55,902 DEBUG [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-12-04T15:44:55,902 DEBUG [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-12-04T15:44:55,902 DEBUG [StoreFileCloser-cee5398235ad5e94b1f3bbb5f1071cfe-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/default/testBlockEvictionOnGracefulStop1733327093098/cee5398235ad5e94b1f3bbb5f1071cfe/family/b36d40b7dadc4cf0b4ca854bb9c05698
2024-12-04T15:44:55,902 INFO  [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.21 KB heapSize=2.80 KB
2024-12-04T15:44:55,907 DEBUG [RS_CLOSE_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/default/testBlockEvictionOnGracefulStop1733327093098/cee5398235ad5e94b1f3bbb5f1071cfe/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9
2024-12-04T15:44:55,907 INFO  [RS_CLOSE_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733327093098,,1733327093101.cee5398235ad5e94b1f3bbb5f1071cfe.
2024-12-04T15:44:55,908 DEBUG [RS_CLOSE_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for cee5398235ad5e94b1f3bbb5f1071cfe:
Waiting for close lock at 1733327095901Running coprocessor pre-close hooks at 1733327095901Disabling compacts and flushes for region at 1733327095901Disabling writes for close at 1733327095901Writing region close event to WAL at 1733327095902 (+1 ms)Running coprocessor post-close hooks at 1733327095907 (+5 ms)Closed at 1733327095907
2024-12-04T15:44:55,908 DEBUG [RS_CLOSE_REGION-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1733327093098,,1733327093101.cee5398235ad5e94b1f3bbb5f1071cfe.
2024-12-04T15:44:55,913 DEBUG [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8d9ca2541cb14ecaa92a56bed30ce115
2024-12-04T15:44:55,929 DEBUG [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/.tmp/info/8d9ca2541cb14ecaa92a56bed30ce115 is 209, key is testBlockEvictionOnGracefulStop1733327093098,,1733327093101.cee5398235ad5e94b1f3bbb5f1071cfe./info:regioninfo/1733327095827/Put/seqid=0
2024-12-04T15:44:55,930 INFO  [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.21 KB at sequenceid=19 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/.tmp/info/8d9ca2541cb14ecaa92a56bed30ce115
2024-12-04T15:44:55,933 DEBUG [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/.tmp/info/8d9ca2541cb14ecaa92a56bed30ce115 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/info/8d9ca2541cb14ecaa92a56bed30ce115
2024-12-04T15:44:55,935 DEBUG [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/info/8d9ca2541cb14ecaa92a56bed30ce115, delay=930 ms
2024-12-04T15:44:55,936 INFO  [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/info/8d9ca2541cb14ecaa92a56bed30ce115, entries=8, sequenceid=19, filesize=6.6 K
2024-12-04T15:44:55,937 INFO  [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 35ms, sequenceid=19, compaction requested=false
2024-12-04T15:44:55,938 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/info/894dec8ada2147278c74effb2bcef058
2024-12-04T15:44:55,938 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/info/8d9ca2541cb14ecaa92a56bed30ce115
2024-12-04T15:44:55,939 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/ns/f23add35bb664e8c8ffd31ceff59860f
2024-12-04T15:44:55,939 INFO  [regionserver/a21b6491b371:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-04T15:44:55,939 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/table/362037b9e197444ab19a3bb983bd0117
2024-12-04T15:44:55,944 DEBUG [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/data/hbase/meta/1588230740/recovered.edits/22.seqid, newMaxSeqId=22, maxSeqId=14
2024-12-04T15:44:55,945 DEBUG [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-12-04T15:44:55,945 INFO  [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-12-04T15:44:55,945 DEBUG [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1733327095902Running coprocessor pre-close hooks at 1733327095902Disabling compacts and flushes for region at 1733327095902Disabling writes for close at 1733327095902Obtaining lock to block concurrent updates at 1733327095902Preparing flush snapshotting stores in 1588230740 at 1733327095902Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1238, getHeapSize=2800, getOffHeapSize=0, getCellsCount=8 at 1733327095903 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733327095903Flushing 1588230740/info: creating writer at 1733327095903Flushing 1588230740/info: appending metadata at 1733327095928 (+25 ms)Flushing 1588230740/info: closing flushed file at 1733327095928Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65805985: reopening flushed file at 1733327095933 (+5 ms)Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 35ms, sequenceid=19, compaction requested=false at 1733327095937 (+4 ms)Writing region close event to WAL at 1733327095940 (+3 ms)Running coprocessor post-close hooks at 1733327095945 (+5 ms)Closed at 1733327095945
2024-12-04T15:44:55,945 DEBUG [RS_CLOSE_META-regionserver/a21b6491b371:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-12-04T15:44:55,948 INFO  [regionserver/a21b6491b371:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-12-04T15:44:56,102 INFO  [RS:2;a21b6491b371:38071 {}] regionserver.HRegionServer(976): stopping server a21b6491b371,38071,1733327095293; all regions closed.
2024-12-04T15:44:56,102 INFO  [RS:1;a21b6491b371:34119 {}] regionserver.HRegionServer(976): stopping server a21b6491b371,34119,1733327092157; all regions closed.
2024-12-04T15:44:56,104 DEBUG [RS:2;a21b6491b371:38071 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/oldWALs
2024-12-04T15:44:56,104 INFO  [RS:2;a21b6491b371:38071 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a21b6491b371%2C38071%2C1733327095293:(num 1733327095485)
2024-12-04T15:44:56,104 DEBUG [RS:1;a21b6491b371:34119 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/oldWALs
2024-12-04T15:44:56,104 DEBUG [RS:2;a21b6491b371:38071 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-04T15:44:56,104 INFO  [RS:2;a21b6491b371:38071 {}] regionserver.LeaseManager(133): Closed leases
2024-12-04T15:44:56,104 INFO  [RS:1;a21b6491b371:34119 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a21b6491b371%2C34119%2C1733327092157.meta:.meta(num 1733327095331)
2024-12-04T15:44:56,104 INFO  [RS:2;a21b6491b371:38071 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-04T15:44:56,104 INFO  [RS:2;a21b6491b371:38071 {}] hbase.ChoreService(370): Chore service for: regionserver/a21b6491b371:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-12-04T15:44:56,104 INFO  [RS:2;a21b6491b371:38071 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-12-04T15:44:56,104 INFO  [RS:2;a21b6491b371:38071 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-12-04T15:44:56,104 INFO  [RS:2;a21b6491b371:38071 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-12-04T15:44:56,105 INFO  [RS:2;a21b6491b371:38071 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-04T15:44:56,104 INFO  [regionserver/a21b6491b371:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-04T15:44:56,105 INFO  [RS:2;a21b6491b371:38071 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38071
2024-12-04T15:44:56,106 DEBUG [RS:1;a21b6491b371:34119 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/oldWALs
2024-12-04T15:44:56,106 INFO  [RS:1;a21b6491b371:34119 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a21b6491b371%2C34119%2C1733327092157:(num 1733327092677)
2024-12-04T15:44:56,106 DEBUG [RS:1;a21b6491b371:34119 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-12-04T15:44:56,106 INFO  [RS:1;a21b6491b371:34119 {}] regionserver.LeaseManager(133): Closed leases
2024-12-04T15:44:56,106 INFO  [RS:1;a21b6491b371:34119 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-04T15:44:56,106 INFO  [RS:1;a21b6491b371:34119 {}] hbase.ChoreService(370): Chore service for: regionserver/a21b6491b371:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-12-04T15:44:56,106 INFO  [RS:1;a21b6491b371:34119 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-04T15:44:56,107 INFO  [regionserver/a21b6491b371:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-04T15:44:56,107 INFO  [RS:1;a21b6491b371:34119 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34119
2024-12-04T15:44:56,107 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44449-0x1017e2663e10000, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-12-04T15:44:56,107 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38071-0x1017e2663e10003, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a21b6491b371,38071,1733327095293
2024-12-04T15:44:56,107 INFO  [RS:2;a21b6491b371:38071 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-04T15:44:56,110 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34119-0x1017e2663e10002, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a21b6491b371,34119,1733327092157
2024-12-04T15:44:56,110 INFO  [RS:1;a21b6491b371:34119 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-04T15:44:56,110 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a21b6491b371,38071,1733327095293]
2024-12-04T15:44:56,112 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a21b6491b371,38071,1733327095293 already deleted, retry=false
2024-12-04T15:44:56,112 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a21b6491b371,38071,1733327095293 expired; onlineServers=1
2024-12-04T15:44:56,112 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a21b6491b371,34119,1733327092157]
2024-12-04T15:44:56,113 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a21b6491b371,34119,1733327092157 already deleted, retry=false
2024-12-04T15:44:56,113 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a21b6491b371,34119,1733327092157 expired; onlineServers=0
2024-12-04T15:44:56,113 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'a21b6491b371,44449,1733327092066' *****
2024-12-04T15:44:56,113 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-12-04T15:44:56,113 INFO  [M:0;a21b6491b371:44449 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-12-04T15:44:56,113 INFO  [M:0;a21b6491b371:44449 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-12-04T15:44:56,114 DEBUG [M:0;a21b6491b371:44449 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-12-04T15:44:56,114 DEBUG [M:0;a21b6491b371:44449 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-12-04T15:44:56,114 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-12-04T15:44:56,114 DEBUG [master/a21b6491b371:0:becomeActiveMaster-HFileCleaner.large.0-1733327092403 {}] cleaner.HFileCleaner(306): Exit Thread[master/a21b6491b371:0:becomeActiveMaster-HFileCleaner.large.0-1733327092403,5,FailOnTimeoutGroup]
2024-12-04T15:44:56,114 DEBUG [master/a21b6491b371:0:becomeActiveMaster-HFileCleaner.small.0-1733327092403 {}] cleaner.HFileCleaner(306): Exit Thread[master/a21b6491b371:0:becomeActiveMaster-HFileCleaner.small.0-1733327092403,5,FailOnTimeoutGroup]
2024-12-04T15:44:56,114 INFO  [M:0;a21b6491b371:44449 {}] hbase.ChoreService(370): Chore service for: master/a21b6491b371:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-12-04T15:44:56,114 INFO  [M:0;a21b6491b371:44449 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-12-04T15:44:56,114 DEBUG [M:0;a21b6491b371:44449 {}] master.HMaster(1795): Stopping service threads
2024-12-04T15:44:56,114 INFO  [M:0;a21b6491b371:44449 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-12-04T15:44:56,115 INFO  [M:0;a21b6491b371:44449 {}] procedure2.ProcedureExecutor(723): Stopping
2024-12-04T15:44:56,115 INFO  [M:0;a21b6491b371:44449 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-12-04T15:44:56,115 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-12-04T15:44:56,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44449-0x1017e2663e10000, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-12-04T15:44:56,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44449-0x1017e2663e10000, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-12-04T15:44:56,117 DEBUG [M:0;a21b6491b371:44449 {}] zookeeper.ZKUtil(347): master:44449-0x1017e2663e10000, quorum=127.0.0.1:58881, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-12-04T15:44:56,117 WARN  [M:0;a21b6491b371:44449 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-12-04T15:44:56,117 INFO  [M:0;a21b6491b371:44449 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/.lastflushedseqids
2024-12-04T15:44:56,121 INFO  [M:0;a21b6491b371:44449 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-12-04T15:44:56,121 INFO  [M:0;a21b6491b371:44449 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-12-04T15:44:56,122 DEBUG [M:0;a21b6491b371:44449 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-12-04T15:44:56,122 INFO  [M:0;a21b6491b371:44449 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-04T15:44:56,122 DEBUG [M:0;a21b6491b371:44449 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-04T15:44:56,122 DEBUG [M:0;a21b6491b371:44449 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-12-04T15:44:56,122 DEBUG [M:0;a21b6491b371:44449 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-04T15:44:56,122 INFO  [M:0;a21b6491b371:44449 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=39.50 KB heapSize=49.91 KB
2024-12-04T15:44:56,133 DEBUG [M:0;a21b6491b371:44449 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 4fa7a20a7cc24c56a44d1e3aeadc0540
2024-12-04T15:44:56,148 DEBUG [M:0;a21b6491b371:44449 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4fa7a20a7cc24c56a44d1e3aeadc0540 is 82, key is hbase:meta,,1/info:regioninfo/1733327095376/Put/seqid=0
2024-12-04T15:44:56,148 INFO  [M:0;a21b6491b371:44449 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4fa7a20a7cc24c56a44d1e3aeadc0540
2024-12-04T15:44:56,158 DEBUG [M:0;a21b6491b371:44449 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 2bf2af50fe4d4b9d9aa61758265da280
2024-12-04T15:44:56,173 DEBUG [M:0;a21b6491b371:44449 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2bf2af50fe4d4b9d9aa61758265da280 is 782, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733327093480/Put/seqid=0
2024-12-04T15:44:56,174 INFO  [M:0;a21b6491b371:44449 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.27 KB at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2bf2af50fe4d4b9d9aa61758265da280
2024-12-04T15:44:56,182 DEBUG [M:0;a21b6491b371:44449 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on e44649fc99d945e599f7bbc455243d8b
2024-12-04T15:44:56,197 DEBUG [M:0;a21b6491b371:44449 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e44649fc99d945e599f7bbc455243d8b is 69, key is a21b6491b371,34119,1733327092157/rs:state/1733327092513/Put/seqid=0
2024-12-04T15:44:56,198 INFO  [M:0;a21b6491b371:44449 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e44649fc99d945e599f7bbc455243d8b
2024-12-04T15:44:56,199 INFO  [M:0;a21b6491b371:44449 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e44649fc99d945e599f7bbc455243d8b
2024-12-04T15:44:56,200 DEBUG [M:0;a21b6491b371:44449 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4fa7a20a7cc24c56a44d1e3aeadc0540 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/4fa7a20a7cc24c56a44d1e3aeadc0540
2024-12-04T15:44:56,201 INFO  [M:0;a21b6491b371:44449 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/4fa7a20a7cc24c56a44d1e3aeadc0540, entries=14, sequenceid=102, filesize=5.9 K
2024-12-04T15:44:56,202 DEBUG [M:0;a21b6491b371:44449 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2bf2af50fe4d4b9d9aa61758265da280 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2bf2af50fe4d4b9d9aa61758265da280
2024-12-04T15:44:56,204 INFO  [M:0;a21b6491b371:44449 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2bf2af50fe4d4b9d9aa61758265da280, entries=11, sequenceid=102, filesize=6.5 K
2024-12-04T15:44:56,204 DEBUG [M:0;a21b6491b371:44449 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e44649fc99d945e599f7bbc455243d8b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e44649fc99d945e599f7bbc455243d8b
2024-12-04T15:44:56,206 INFO  [M:0;a21b6491b371:44449 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e44649fc99d945e599f7bbc455243d8b
2024-12-04T15:44:56,206 INFO  [M:0;a21b6491b371:44449 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c66093bd-1ece-8753-fb51-bc500b438ae4/ae33cf15-7247-74bc-aaf8-64e1b2716621/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e44649fc99d945e599f7bbc455243d8b, entries=3, sequenceid=102, filesize=5.3 K
2024-12-04T15:44:56,207 INFO  [M:0;a21b6491b371:44449 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.50 KB/40448, heapSize ~49.61 KB/50800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 85ms, sequenceid=102, compaction requested=false
2024-12-04T15:44:56,209 INFO  [M:0;a21b6491b371:44449 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-12-04T15:44:56,209 DEBUG [M:0;a21b6491b371:44449 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1733327096122Disabling compacts and flushes for region at 1733327096122Disabling writes for close at 1733327096122Obtaining lock to block concurrent updates at 1733327096122Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733327096122Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=40448, getHeapSize=51040, getOffHeapSize=0, getCellsCount=127 at 1733327096123 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733327096123Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733327096123Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733327096147 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733327096147Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733327096151 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733327096172 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733327096173 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733327096176 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733327096196 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733327096196Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c5845ad: reopening flushed file at 1733327096199 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d177d0a: reopening flushed file at 1733327096201 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3656895: reopening flushed file at 1733327096204 (+3 ms)Finished flush of dataSize ~39.50 KB/40448, heapSize ~49.61 KB/50800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 85ms, sequenceid=102, compaction requested=false at 1733327096207 (+3 ms)Writing region close event to WAL at 1733327096209 (+2 ms)Closed at 1733327096209
2024-12-04T15:44:56,210 INFO  [M:0;a21b6491b371:44449 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-12-04T15:44:56,210 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-12-04T15:44:56,210 INFO  [M:0;a21b6491b371:44449 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44449
2024-12-04T15:44:56,210 INFO  [M:0;a21b6491b371:44449 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-12-04T15:44:56,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38071-0x1017e2663e10003, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-04T15:44:56,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38071-0x1017e2663e10003, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-04T15:44:56,211 INFO  [RS:2;a21b6491b371:38071 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-04T15:44:56,211 INFO  [RS:2;a21b6491b371:38071 {}] regionserver.HRegionServer(1031): Exiting; stopping=a21b6491b371,38071,1733327095293; zookeeper connection closed.
2024-12-04T15:44:56,211 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@733b299f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@733b299f
2024-12-04T15:44:56,212 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34119-0x1017e2663e10002, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-04T15:44:56,212 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34119-0x1017e2663e10002, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-04T15:44:56,212 INFO  [RS:1;a21b6491b371:34119 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-04T15:44:56,212 INFO  [RS:1;a21b6491b371:34119 {}] regionserver.HRegionServer(1031): Exiting; stopping=a21b6491b371,34119,1733327092157; zookeeper connection closed.
2024-12-04T15:44:56,212 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1f99475f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1f99475f
2024-12-04T15:44:56,212 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete
2024-12-04T15:44:56,313 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44449-0x1017e2663e10000, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-04T15:44:56,313 INFO  [M:0;a21b6491b371:44449 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-12-04T15:44:56,313 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44449-0x1017e2663e10000, quorum=127.0.0.1:58881, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-12-04T15:44:56,317 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-12-04T15:44:56,319 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-12-04T15:44:56,319 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-12-04T15:44:56,326 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=70 (was 49)
Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-6-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-6-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-6-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=846 (was 902), ProcessCount=11 (was 11), AvailableMemoryMB=9815 (was 9910)