2024-11-24 03:55:06,054 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-24 03:55:06,074 main DEBUG Took 0.018163 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-24 03:55:06,075 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-24 03:55:06,075 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-24 03:55:06,076 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-24 03:55:06,077 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 03:55:06,086 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-24 03:55:06,101 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 03:55:06,102 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 03:55:06,103 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 03:55:06,104 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 03:55:06,105 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 03:55:06,105 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 03:55:06,107 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 03:55:06,107 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 03:55:06,108 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 03:55:06,109 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 03:55:06,110 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 03:55:06,111 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 03:55:06,112 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 03:55:06,112 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 03:55:06,112 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 03:55:06,113 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 03:55:06,113 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 03:55:06,114 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 03:55:06,114 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 03:55:06,114 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 03:55:06,115 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 03:55:06,115 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 03:55:06,115 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 03:55:06,116 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 03:55:06,116 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 03:55:06,116 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-24 03:55:06,118 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 03:55:06,119 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-24 03:55:06,121 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-24 03:55:06,121 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-24 03:55:06,123 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-24 03:55:06,123 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-24 03:55:06,131 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-24 03:55:06,133 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-24 03:55:06,135 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-24 03:55:06,135 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-24 03:55:06,136 main DEBUG createAppenders(={Console}) 2024-11-24 03:55:06,137 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-24 03:55:06,137 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-24 03:55:06,137 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-24 03:55:06,138 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-24 03:55:06,138 main DEBUG OutputStream closed 2024-11-24 03:55:06,138 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-24 03:55:06,139 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-24 03:55:06,139 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-24 03:55:06,210 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-24 03:55:06,212 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-24 03:55:06,213 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-24 03:55:06,215 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-24 03:55:06,215 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-24 03:55:06,216 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-24 03:55:06,216 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-24 03:55:06,216 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-24 03:55:06,216 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-24 03:55:06,217 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-24 03:55:06,217 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-24 03:55:06,217 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-24 03:55:06,218 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-24 03:55:06,218 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-24 03:55:06,218 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-24 03:55:06,218 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-24 03:55:06,219 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-24 03:55:06,220 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-24 03:55:06,222 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24 03:55:06,222 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-24 03:55:06,223 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-24 03:55:06,223 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-24T03:55:06,236 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-24 03:55:06,238 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-24 03:55:06,239 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24T03:55:06,464 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17 2024-11-24T03:55:06,490 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=256, ProcessCount=11, AvailableMemoryMB=8248 2024-11-24T03:55:06,599 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-24T03:55:06,685 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T03:55:06,698 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/cluster_42e1fe1f-5218-9482-90d4-5e9c3304db41, deleteOnExit=true 2024-11-24T03:55:06,775 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/cluster_42e1fe1f-5218-9482-90d4-5e9c3304db41/zookeeper_0, clientPort=59166, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/cluster_42e1fe1f-5218-9482-90d4-5e9c3304db41/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/cluster_42e1fe1f-5218-9482-90d4-5e9c3304db41/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-24T03:55:06,786 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59166 2024-11-24T03:55:06,787 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T03:55:06,862 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b with version=8 2024-11-24T03:55:06,862 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/hbase-staging 2024-11-24T03:55:06,956 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-24T03:55:07,198 INFO [Time-limited test {}] client.ConnectionUtils(128): master/71d8d2d6408d:0 server-side Connection retries=45 2024-11-24T03:55:07,207 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T03:55:07,208 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T03:55:07,218 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T03:55:07,218 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T03:55:07,218 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T03:55:07,350 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-24T03:55:07,427 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T03:55:07,461 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-24T03:55:07,471 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-24T03:55:07,475 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T03:55:07,513 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 94022 (auto-detected) 2024-11-24T03:55:07,514 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-24T03:55:07,535 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40957 2024-11-24T03:55:07,565 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40957 connecting to ZooKeeper ensemble=127.0.0.1:59166 2024-11-24T03:55:07,691 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:409570x0, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T03:55:07,695 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40957-0x1016c44a74a0000 connected 2024-11-24T03:55:07,796 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T03:55:07,798 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T03:55:07,806 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40957-0x1016c44a74a0000, quorum=127.0.0.1:59166, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T03:55:07,809 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b, hbase.cluster.distributed=false 2024-11-24T03:55:07,834 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40957-0x1016c44a74a0000, quorum=127.0.0.1:59166, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T03:55:07,839 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40957 2024-11-24T03:55:07,840 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40957 2024-11-24T03:55:07,840 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40957 2024-11-24T03:55:07,841 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40957 2024-11-24T03:55:07,841 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40957 2024-11-24T03:55:07,917 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/71d8d2d6408d:0 server-side Connection retries=45 2024-11-24T03:55:07,919 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T03:55:07,919 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T03:55:07,919 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T03:55:07,920 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T03:55:07,920 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T03:55:07,922 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T03:55:07,925 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T03:55:07,926 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46457 2024-11-24T03:55:07,928 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46457 connecting to ZooKeeper ensemble=127.0.0.1:59166 2024-11-24T03:55:07,929 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T03:55:07,930 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T03:55:07,947 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:464570x0, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T03:55:07,948 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46457-0x1016c44a74a0001 connected 2024-11-24T03:55:07,948 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46457-0x1016c44a74a0001, quorum=127.0.0.1:59166, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T03:55:07,956 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T03:55:07,969 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/bucket.cache 2024-11-24T03:55:07,975 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-24T03:55:07,977 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-24T03:55:07,979 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-24T03:55:07,980 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-24T03:55:07,981 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-24T03:55:07,988 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-24T03:55:07,993 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-24T03:55:07,993 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/bucket.persistence is complete. 2024-11-24T03:55:07,994 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T03:55:07,997 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46457-0x1016c44a74a0001, quorum=127.0.0.1:59166, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T03:55:08,002 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46457-0x1016c44a74a0001, quorum=127.0.0.1:59166, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T03:55:08,003 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46457 2024-11-24T03:55:08,004 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46457 2024-11-24T03:55:08,004 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46457 2024-11-24T03:55:08,010 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46457 2024-11-24T03:55:08,011 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46457 2024-11-24T03:55:08,039 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/71d8d2d6408d:0 server-side Connection retries=45 2024-11-24T03:55:08,039 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T03:55:08,039 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T03:55:08,040 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T03:55:08,040 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T03:55:08,040 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T03:55:08,041 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T03:55:08,041 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T03:55:08,043 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36947 2024-11-24T03:55:08,046 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36947 connecting to ZooKeeper ensemble=127.0.0.1:59166 2024-11-24T03:55:08,047 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T03:55:08,048 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T03:55:08,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:369470x0, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T03:55:08,081 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:369470x0, quorum=127.0.0.1:59166, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T03:55:08,082 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36947-0x1016c44a74a0002 connected 2024-11-24T03:55:08,082 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T03:55:08,083 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/bucket.cache 2024-11-24T03:55:08,083 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-24T03:55:08,084 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-24T03:55:08,084 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-24T03:55:08,084 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-24T03:55:08,085 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-24T03:55:08,085 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-24T03:55:08,086 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-24T03:55:08,086 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/bucket.persistence is complete. 2024-11-24T03:55:08,086 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T03:55:08,088 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36947-0x1016c44a74a0002, quorum=127.0.0.1:59166, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T03:55:08,091 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36947-0x1016c44a74a0002, quorum=127.0.0.1:59166, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T03:55:08,095 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36947 2024-11-24T03:55:08,095 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36947 2024-11-24T03:55:08,096 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36947 2024-11-24T03:55:08,098 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36947 2024-11-24T03:55:08,098 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36947 2024-11-24T03:55:08,120 DEBUG [M:0;71d8d2d6408d:40957 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;71d8d2d6408d:40957 2024-11-24T03:55:08,121 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/71d8d2d6408d,40957,1732420507038 2024-11-24T03:55:08,131 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40957-0x1016c44a74a0000, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T03:55:08,131 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36947-0x1016c44a74a0002, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T03:55:08,131 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46457-0x1016c44a74a0001, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T03:55:08,133 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40957-0x1016c44a74a0000, quorum=127.0.0.1:59166, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/71d8d2d6408d,40957,1732420507038 2024-11-24T03:55:08,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36947-0x1016c44a74a0002, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T03:55:08,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40957-0x1016c44a74a0000, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T03:55:08,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46457-0x1016c44a74a0001, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T03:55:08,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36947-0x1016c44a74a0002, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T03:55:08,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46457-0x1016c44a74a0001, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T03:55:08,165 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40957-0x1016c44a74a0000, quorum=127.0.0.1:59166, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-24T03:55:08,167 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/71d8d2d6408d,40957,1732420507038 from backup master directory 2024-11-24T03:55:08,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36947-0x1016c44a74a0002, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T03:55:08,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40957-0x1016c44a74a0000, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/71d8d2d6408d,40957,1732420507038 2024-11-24T03:55:08,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46457-0x1016c44a74a0001, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T03:55:08,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40957-0x1016c44a74a0000, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T03:55:08,181 WARN [master/71d8d2d6408d:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T03:55:08,181 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=71d8d2d6408d,40957,1732420507038 2024-11-24T03:55:08,183 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-24T03:55:08,185 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-24T03:55:08,195 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/hbase.id] with ID: 5ef32ff8-f46c-42b9-80e5-b7f373e9c523 2024-11-24T03:55:08,195 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/.tmp/hbase.id 2024-11-24T03:55:08,201 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/hbase.id] 2024-11-24T03:55:08,241 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T03:55:08,246 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-24T03:55:08,272 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 23ms. 2024-11-24T03:55:08,305 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46457-0x1016c44a74a0001, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T03:55:08,305 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40957-0x1016c44a74a0000, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T03:55:08,305 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36947-0x1016c44a74a0002, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T03:55:08,330 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T03:55:08,333 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-24T03:55:08,375 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T03:55:08,382 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T03:55:08,430 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/MasterData/data/master/store 2024-11-24T03:55:08,449 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-24T03:55:08,453 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T03:55:08,455 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T03:55:08,455 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T03:55:08,455 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T03:55:08,457 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T03:55:08,457 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T03:55:08,457 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T03:55:08,458 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732420508455Disabling compacts and flushes for region at 1732420508455Disabling writes for close at 1732420508457 (+2 ms)Writing region close event to WAL at 1732420508457Closed at 1732420508457 2024-11-24T03:55:08,459 WARN [master/71d8d2d6408d:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/MasterData/data/master/store/.initializing 2024-11-24T03:55:08,459 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/MasterData/WALs/71d8d2d6408d,40957,1732420507038 2024-11-24T03:55:08,471 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T03:55:08,484 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=71d8d2d6408d%2C40957%2C1732420507038, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/MasterData/WALs/71d8d2d6408d,40957,1732420507038, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/MasterData/oldWALs, maxLogs=10 2024-11-24T03:55:08,517 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/MasterData/WALs/71d8d2d6408d,40957,1732420507038/71d8d2d6408d%2C40957%2C1732420507038.1732420508488 2024-11-24T03:55:08,518 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T03:55:08,519 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-24T03:55:08,520 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T03:55:08,523 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T03:55:08,525 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T03:55:08,565 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T03:55:08,593 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-24T03:55:08,598 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T03:55:08,599 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T03:55:08,600 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T03:55:08,604 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-24T03:55:08,605 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T03:55:08,605 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T03:55:08,605 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T03:55:08,609 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-24T03:55:08,610 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T03:55:08,610 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T03:55:08,610 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T03:55:08,615 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-24T03:55:08,615 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T03:55:08,615 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T03:55:08,616 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T03:55:08,619 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T03:55:08,620 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T03:55:08,623 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T03:55:08,624 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T03:55:08,627 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T03:55:08,629 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T03:55:08,636 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T03:55:08,637 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68186322, jitterRate=0.016055375337600708}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T03:55:08,647 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732420508537Initializing all the Stores at 1732420508537Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732420508538 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732420508539 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732420508540 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732420508540Cleaning up temporary data from old regions at 1732420508624 (+84 ms)Region opened successfully at 1732420508647 (+23 ms) 2024-11-24T03:55:08,649 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-24T03:55:08,688 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e1af41e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=71d8d2d6408d/172.17.0.2:0 2024-11-24T03:55:08,732 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-24T03:55:08,748 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-24T03:55:08,748 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-24T03:55:08,752 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T03:55:08,753 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T03:55:08,761 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-24T03:55:08,761 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-24T03:55:08,793 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-24T03:55:08,803 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40957-0x1016c44a74a0000, quorum=127.0.0.1:59166, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-24T03:55:08,855 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-24T03:55:08,858 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-24T03:55:08,860 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40957-0x1016c44a74a0000, quorum=127.0.0.1:59166, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-24T03:55:08,871 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-24T03:55:08,873 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-24T03:55:08,879 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40957-0x1016c44a74a0000, quorum=127.0.0.1:59166, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-24T03:55:08,888 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-24T03:55:08,890 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40957-0x1016c44a74a0000, quorum=127.0.0.1:59166, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-24T03:55:08,896 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-24T03:55:08,913 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40957-0x1016c44a74a0000, quorum=127.0.0.1:59166, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-24T03:55:08,921 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-24T03:55:08,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36947-0x1016c44a74a0002, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T03:55:08,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40957-0x1016c44a74a0000, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T03:55:08,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46457-0x1016c44a74a0001, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T03:55:08,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40957-0x1016c44a74a0000, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T03:55:08,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46457-0x1016c44a74a0001, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T03:55:08,939 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36947-0x1016c44a74a0002, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T03:55:08,942 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=71d8d2d6408d,40957,1732420507038, sessionid=0x1016c44a74a0000, setting cluster-up flag (Was=false) 2024-11-24T03:55:08,963 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36947-0x1016c44a74a0002, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T03:55:08,963 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46457-0x1016c44a74a0001, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T03:55:08,963 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40957-0x1016c44a74a0000, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T03:55:08,988 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-24T03:55:08,991 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=71d8d2d6408d,40957,1732420507038 2024-11-24T03:55:09,013 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46457-0x1016c44a74a0001, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T03:55:09,013 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36947-0x1016c44a74a0002, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T03:55:09,013 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40957-0x1016c44a74a0000, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T03:55:09,038 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-24T03:55:09,041 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=71d8d2d6408d,40957,1732420507038 2024-11-24T03:55:09,046 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-24T03:55:09,103 INFO [RS:0;71d8d2d6408d:46457 {}] regionserver.HRegionServer(746): ClusterId : 5ef32ff8-f46c-42b9-80e5-b7f373e9c523 2024-11-24T03:55:09,103 INFO [RS:1;71d8d2d6408d:36947 {}] regionserver.HRegionServer(746): ClusterId : 5ef32ff8-f46c-42b9-80e5-b7f373e9c523 2024-11-24T03:55:09,106 DEBUG [RS:1;71d8d2d6408d:36947 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T03:55:09,106 DEBUG [RS:0;71d8d2d6408d:46457 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T03:55:09,122 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T03:55:09,131 DEBUG [RS:1;71d8d2d6408d:36947 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T03:55:09,131 DEBUG [RS:0;71d8d2d6408d:46457 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T03:55:09,131 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-24T03:55:09,132 DEBUG [RS:1;71d8d2d6408d:36947 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T03:55:09,132 DEBUG [RS:0;71d8d2d6408d:46457 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T03:55:09,140 DEBUG [RS:0;71d8d2d6408d:46457 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T03:55:09,140 DEBUG [RS:1;71d8d2d6408d:36947 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T03:55:09,140 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-24T03:55:09,140 DEBUG [RS:1;71d8d2d6408d:36947 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dd85e38, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=71d8d2d6408d/172.17.0.2:0 2024-11-24T03:55:09,140 DEBUG [RS:0;71d8d2d6408d:46457 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d55d44b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=71d8d2d6408d/172.17.0.2:0 2024-11-24T03:55:09,147 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 71d8d2d6408d,40957,1732420507038 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-24T03:55:09,155 DEBUG [RS:0;71d8d2d6408d:46457 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;71d8d2d6408d:46457 2024-11-24T03:55:09,156 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/71d8d2d6408d:0, corePoolSize=5, maxPoolSize=5 2024-11-24T03:55:09,156 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/71d8d2d6408d:0, corePoolSize=5, maxPoolSize=5 2024-11-24T03:55:09,157 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/71d8d2d6408d:0, corePoolSize=5, maxPoolSize=5 2024-11-24T03:55:09,157 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/71d8d2d6408d:0, corePoolSize=5, maxPoolSize=5 2024-11-24T03:55:09,157 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/71d8d2d6408d:0, corePoolSize=10, maxPoolSize=10 2024-11-24T03:55:09,157 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:09,157 DEBUG [RS:1;71d8d2d6408d:36947 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;71d8d2d6408d:36947 2024-11-24T03:55:09,157 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/71d8d2d6408d:0, corePoolSize=2, maxPoolSize=2 2024-11-24T03:55:09,158 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:09,159 INFO [RS:0;71d8d2d6408d:46457 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T03:55:09,159 INFO [RS:1;71d8d2d6408d:36947 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T03:55:09,159 INFO [RS:0;71d8d2d6408d:46457 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T03:55:09,159 INFO [RS:1;71d8d2d6408d:36947 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T03:55:09,159 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732420539159 2024-11-24T03:55:09,159 DEBUG [RS:0;71d8d2d6408d:46457 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T03:55:09,159 DEBUG [RS:1;71d8d2d6408d:36947 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T03:55:09,161 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-24T03:55:09,162 INFO [RS:1;71d8d2d6408d:36947 {}] regionserver.HRegionServer(2659): reportForDuty to master=71d8d2d6408d,40957,1732420507038 with port=36947, startcode=1732420508038 2024-11-24T03:55:09,162 INFO [RS:0;71d8d2d6408d:46457 {}] regionserver.HRegionServer(2659): reportForDuty to master=71d8d2d6408d,40957,1732420507038 with port=46457, startcode=1732420507885 2024-11-24T03:55:09,163 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-24T03:55:09,164 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T03:55:09,164 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-24T03:55:09,168 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-24T03:55:09,168 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-24T03:55:09,169 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-24T03:55:09,169 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-24T03:55:09,170 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T03:55:09,170 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-24T03:55:09,170 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:09,177 DEBUG [RS:0;71d8d2d6408d:46457 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T03:55:09,177 DEBUG [RS:1;71d8d2d6408d:36947 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T03:55:09,180 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-24T03:55:09,182 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-24T03:55:09,182 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-24T03:55:09,187 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-24T03:55:09,188 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-24T03:55:09,190 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/71d8d2d6408d:0:becomeActiveMaster-HFileCleaner.large.0-1732420509189,5,FailOnTimeoutGroup] 2024-11-24T03:55:09,191 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/71d8d2d6408d:0:becomeActiveMaster-HFileCleaner.small.0-1732420509190,5,FailOnTimeoutGroup] 2024-11-24T03:55:09,191 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:09,191 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-24T03:55:09,192 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:09,192 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:09,196 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-24T03:55:09,196 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b 2024-11-24T03:55:09,205 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T03:55:09,206 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-24T03:55:09,210 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T03:55:09,210 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T03:55:09,211 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T03:55:09,211 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-24T03:55:09,223 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T03:55:09,223 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T03:55:09,224 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T03:55:09,224 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-24T03:55:09,236 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T03:55:09,236 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T03:55:09,237 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T03:55:09,237 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-24T03:55:09,244 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T03:55:09,244 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T03:55:09,245 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T03:55:09,245 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T03:55:09,247 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740 2024-11-24T03:55:09,247 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740 2024-11-24T03:55:09,247 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48363, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T03:55:09,247 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58219, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T03:55:09,247 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T03:55:09,248 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T03:55:09,248 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T03:55:09,249 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T03:55:09,254 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40957 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 71d8d2d6408d,36947,1732420508038 2024-11-24T03:55:09,257 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40957 {}] master.ServerManager(517): Registering regionserver=71d8d2d6408d,36947,1732420508038 2024-11-24T03:55:09,257 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T03:55:09,258 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60427436, jitterRate=-0.09956103563308716}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T03:55:09,262 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732420509205Initializing all the Stores at 1732420509205Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732420509206 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732420509206Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732420509206Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732420509206Cleaning up temporary data from old regions at 1732420509248 (+42 ms)Region opened successfully at 1732420509261 (+13 ms) 2024-11-24T03:55:09,262 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T03:55:09,262 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T03:55:09,262 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T03:55:09,262 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T03:55:09,262 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T03:55:09,264 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T03:55:09,264 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732420509262Disabling compacts and flushes for region at 1732420509262Disabling writes for close at 1732420509262Writing region close event to WAL at 1732420509263 (+1 ms)Closed at 1732420509264 (+1 ms) 2024-11-24T03:55:09,266 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T03:55:09,266 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-24T03:55:09,269 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40957 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 71d8d2d6408d,46457,1732420507885 2024-11-24T03:55:09,269 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40957 {}] master.ServerManager(517): Registering regionserver=71d8d2d6408d,46457,1732420507885 2024-11-24T03:55:09,273 DEBUG [RS:1;71d8d2d6408d:36947 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b 2024-11-24T03:55:09,273 DEBUG [RS:1;71d8d2d6408d:36947 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-24T03:55:09,273 DEBUG [RS:1;71d8d2d6408d:36947 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T03:55:09,274 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-24T03:55:09,274 DEBUG [RS:0;71d8d2d6408d:46457 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b 2024-11-24T03:55:09,275 DEBUG [RS:0;71d8d2d6408d:46457 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-24T03:55:09,275 DEBUG [RS:0;71d8d2d6408d:46457 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T03:55:09,282 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T03:55:09,284 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-24T03:55:09,296 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40957-0x1016c44a74a0000, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T03:55:09,322 DEBUG [RS:1;71d8d2d6408d:36947 {}] zookeeper.ZKUtil(111): regionserver:36947-0x1016c44a74a0002, quorum=127.0.0.1:59166, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/71d8d2d6408d,36947,1732420508038 2024-11-24T03:55:09,323 WARN [RS:1;71d8d2d6408d:36947 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T03:55:09,323 INFO [RS:1;71d8d2d6408d:36947 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T03:55:09,323 DEBUG [RS:0;71d8d2d6408d:46457 {}] zookeeper.ZKUtil(111): regionserver:46457-0x1016c44a74a0001, quorum=127.0.0.1:59166, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/71d8d2d6408d,46457,1732420507885 2024-11-24T03:55:09,323 WARN [RS:0;71d8d2d6408d:46457 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T03:55:09,323 DEBUG [RS:1;71d8d2d6408d:36947 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/WALs/71d8d2d6408d,36947,1732420508038 2024-11-24T03:55:09,323 INFO [RS:0;71d8d2d6408d:46457 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T03:55:09,323 DEBUG [RS:0;71d8d2d6408d:46457 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/WALs/71d8d2d6408d,46457,1732420507885 2024-11-24T03:55:09,324 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [71d8d2d6408d,36947,1732420508038] 2024-11-24T03:55:09,325 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [71d8d2d6408d,46457,1732420507885] 2024-11-24T03:55:09,348 INFO [RS:0;71d8d2d6408d:46457 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T03:55:09,348 INFO [RS:1;71d8d2d6408d:36947 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T03:55:09,367 INFO [RS:0;71d8d2d6408d:46457 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T03:55:09,367 INFO [RS:1;71d8d2d6408d:36947 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T03:55:09,373 INFO [RS:0;71d8d2d6408d:46457 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T03:55:09,373 INFO [RS:1;71d8d2d6408d:36947 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T03:55:09,373 INFO [RS:1;71d8d2d6408d:36947 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:09,373 INFO [RS:0;71d8d2d6408d:46457 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:09,374 INFO [RS:1;71d8d2d6408d:36947 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T03:55:09,374 INFO [RS:0;71d8d2d6408d:46457 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T03:55:09,380 INFO [RS:1;71d8d2d6408d:36947 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T03:55:09,380 INFO [RS:0;71d8d2d6408d:46457 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T03:55:09,382 INFO [RS:1;71d8d2d6408d:36947 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:09,382 INFO [RS:0;71d8d2d6408d:46457 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:09,383 DEBUG [RS:1;71d8d2d6408d:36947 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:09,383 DEBUG [RS:0;71d8d2d6408d:46457 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:09,383 DEBUG [RS:1;71d8d2d6408d:36947 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:09,383 DEBUG [RS:0;71d8d2d6408d:46457 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:09,383 DEBUG [RS:1;71d8d2d6408d:36947 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:09,383 DEBUG [RS:0;71d8d2d6408d:46457 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:09,383 DEBUG [RS:1;71d8d2d6408d:36947 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:09,383 DEBUG [RS:1;71d8d2d6408d:36947 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:09,383 DEBUG [RS:0;71d8d2d6408d:46457 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:09,383 DEBUG [RS:1;71d8d2d6408d:36947 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/71d8d2d6408d:0, corePoolSize=2, maxPoolSize=2 2024-11-24T03:55:09,383 DEBUG [RS:0;71d8d2d6408d:46457 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:09,383 DEBUG [RS:1;71d8d2d6408d:36947 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:09,383 DEBUG [RS:0;71d8d2d6408d:46457 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/71d8d2d6408d:0, corePoolSize=2, maxPoolSize=2 2024-11-24T03:55:09,383 DEBUG [RS:1;71d8d2d6408d:36947 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:09,384 DEBUG [RS:1;71d8d2d6408d:36947 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:09,384 DEBUG [RS:0;71d8d2d6408d:46457 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:09,384 DEBUG [RS:1;71d8d2d6408d:36947 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:09,384 DEBUG [RS:0;71d8d2d6408d:46457 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:09,384 DEBUG [RS:1;71d8d2d6408d:36947 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:09,384 DEBUG [RS:0;71d8d2d6408d:46457 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:09,384 DEBUG [RS:1;71d8d2d6408d:36947 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:09,384 DEBUG [RS:1;71d8d2d6408d:36947 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/71d8d2d6408d:0, corePoolSize=3, maxPoolSize=3 2024-11-24T03:55:09,384 DEBUG [RS:0;71d8d2d6408d:46457 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:09,384 DEBUG [RS:1;71d8d2d6408d:36947 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/71d8d2d6408d:0, corePoolSize=3, maxPoolSize=3 2024-11-24T03:55:09,384 DEBUG [RS:0;71d8d2d6408d:46457 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:09,384 DEBUG [RS:0;71d8d2d6408d:46457 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:09,385 DEBUG [RS:0;71d8d2d6408d:46457 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/71d8d2d6408d:0, corePoolSize=3, maxPoolSize=3 2024-11-24T03:55:09,385 DEBUG [RS:0;71d8d2d6408d:46457 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/71d8d2d6408d:0, corePoolSize=3, maxPoolSize=3 2024-11-24T03:55:09,389 INFO [RS:1;71d8d2d6408d:36947 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:09,389 INFO [RS:1;71d8d2d6408d:36947 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:09,389 INFO [RS:1;71d8d2d6408d:36947 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:09,390 INFO [RS:0;71d8d2d6408d:46457 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:09,390 INFO [RS:1;71d8d2d6408d:36947 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:09,390 INFO [RS:0;71d8d2d6408d:46457 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:09,390 INFO [RS:1;71d8d2d6408d:36947 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:09,390 INFO [RS:0;71d8d2d6408d:46457 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:09,390 INFO [RS:1;71d8d2d6408d:36947 {}] hbase.ChoreService(168): Chore ScheduledChore name=71d8d2d6408d,36947,1732420508038-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T03:55:09,390 INFO [RS:0;71d8d2d6408d:46457 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:09,390 INFO [RS:0;71d8d2d6408d:46457 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:09,390 INFO [RS:0;71d8d2d6408d:46457 {}] hbase.ChoreService(168): Chore ScheduledChore name=71d8d2d6408d,46457,1732420507885-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T03:55:09,418 INFO [RS:1;71d8d2d6408d:36947 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T03:55:09,418 INFO [RS:0;71d8d2d6408d:46457 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T03:55:09,420 INFO [RS:0;71d8d2d6408d:46457 {}] hbase.ChoreService(168): Chore ScheduledChore name=71d8d2d6408d,46457,1732420507885-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:09,420 INFO [RS:1;71d8d2d6408d:36947 {}] hbase.ChoreService(168): Chore ScheduledChore name=71d8d2d6408d,36947,1732420508038-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:09,421 INFO [RS:0;71d8d2d6408d:46457 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:09,421 INFO [RS:1;71d8d2d6408d:36947 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:09,421 INFO [RS:1;71d8d2d6408d:36947 {}] regionserver.Replication(171): 71d8d2d6408d,36947,1732420508038 started 2024-11-24T03:55:09,421 INFO [RS:0;71d8d2d6408d:46457 {}] regionserver.Replication(171): 71d8d2d6408d,46457,1732420507885 started 2024-11-24T03:55:09,436 WARN [71d8d2d6408d:40957 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T03:55:09,440 INFO [RS:1;71d8d2d6408d:36947 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:09,440 INFO [RS:1;71d8d2d6408d:36947 {}] regionserver.HRegionServer(1482): Serving as 71d8d2d6408d,36947,1732420508038, RpcServer on 71d8d2d6408d/172.17.0.2:36947, sessionid=0x1016c44a74a0002 2024-11-24T03:55:09,441 DEBUG [RS:1;71d8d2d6408d:36947 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T03:55:09,441 DEBUG [RS:1;71d8d2d6408d:36947 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 71d8d2d6408d,36947,1732420508038 2024-11-24T03:55:09,441 DEBUG [RS:1;71d8d2d6408d:36947 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '71d8d2d6408d,36947,1732420508038' 2024-11-24T03:55:09,441 DEBUG [RS:1;71d8d2d6408d:36947 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T03:55:09,442 DEBUG [RS:1;71d8d2d6408d:36947 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T03:55:09,443 DEBUG [RS:1;71d8d2d6408d:36947 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T03:55:09,443 DEBUG [RS:1;71d8d2d6408d:36947 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T03:55:09,443 DEBUG [RS:1;71d8d2d6408d:36947 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 71d8d2d6408d,36947,1732420508038 2024-11-24T03:55:09,443 DEBUG [RS:1;71d8d2d6408d:36947 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '71d8d2d6408d,36947,1732420508038' 2024-11-24T03:55:09,443 DEBUG [RS:1;71d8d2d6408d:36947 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T03:55:09,444 DEBUG [RS:1;71d8d2d6408d:36947 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T03:55:09,445 DEBUG [RS:1;71d8d2d6408d:36947 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T03:55:09,445 INFO [RS:1;71d8d2d6408d:36947 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T03:55:09,445 INFO [RS:1;71d8d2d6408d:36947 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T03:55:09,446 INFO [RS:0;71d8d2d6408d:46457 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:09,446 INFO [RS:0;71d8d2d6408d:46457 {}] regionserver.HRegionServer(1482): Serving as 71d8d2d6408d,46457,1732420507885, RpcServer on 71d8d2d6408d/172.17.0.2:46457, sessionid=0x1016c44a74a0001 2024-11-24T03:55:09,447 DEBUG [RS:0;71d8d2d6408d:46457 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T03:55:09,447 DEBUG [RS:0;71d8d2d6408d:46457 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 71d8d2d6408d,46457,1732420507885 2024-11-24T03:55:09,447 DEBUG [RS:0;71d8d2d6408d:46457 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '71d8d2d6408d,46457,1732420507885' 2024-11-24T03:55:09,447 DEBUG [RS:0;71d8d2d6408d:46457 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T03:55:09,448 DEBUG [RS:0;71d8d2d6408d:46457 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T03:55:09,449 DEBUG [RS:0;71d8d2d6408d:46457 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T03:55:09,449 DEBUG [RS:0;71d8d2d6408d:46457 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T03:55:09,449 DEBUG [RS:0;71d8d2d6408d:46457 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 71d8d2d6408d,46457,1732420507885 2024-11-24T03:55:09,449 DEBUG [RS:0;71d8d2d6408d:46457 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '71d8d2d6408d,46457,1732420507885' 2024-11-24T03:55:09,449 DEBUG [RS:0;71d8d2d6408d:46457 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T03:55:09,450 DEBUG [RS:0;71d8d2d6408d:46457 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T03:55:09,450 DEBUG [RS:0;71d8d2d6408d:46457 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T03:55:09,450 INFO [RS:0;71d8d2d6408d:46457 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T03:55:09,450 INFO [RS:0;71d8d2d6408d:46457 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T03:55:09,550 INFO [RS:1;71d8d2d6408d:36947 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T03:55:09,551 INFO [RS:0;71d8d2d6408d:46457 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T03:55:09,552 INFO [RS:1;71d8d2d6408d:36947 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=71d8d2d6408d%2C36947%2C1732420508038, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/WALs/71d8d2d6408d,36947,1732420508038, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/oldWALs, maxLogs=55 2024-11-24T03:55:09,552 INFO [RS:0;71d8d2d6408d:46457 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=71d8d2d6408d%2C46457%2C1732420507885, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/WALs/71d8d2d6408d,46457,1732420507885, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/oldWALs, maxLogs=55 2024-11-24T03:55:09,574 INFO [RS:0;71d8d2d6408d:46457 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/WALs/71d8d2d6408d,46457,1732420507885/71d8d2d6408d%2C46457%2C1732420507885.1732420509557 2024-11-24T03:55:09,574 INFO [RS:1;71d8d2d6408d:36947 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/WALs/71d8d2d6408d,36947,1732420508038/71d8d2d6408d%2C36947%2C1732420508038.1732420509558 2024-11-24T03:55:09,575 DEBUG [RS:0;71d8d2d6408d:46457 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T03:55:09,578 DEBUG [RS:1;71d8d2d6408d:36947 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T03:55:09,692 DEBUG [71d8d2d6408d:40957 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-24T03:55:09,704 DEBUG [71d8d2d6408d:40957 {}] balancer.BalancerClusterState(204): Hosts are {71d8d2d6408d=0} racks are {/default-rack=0} 2024-11-24T03:55:09,707 DEBUG [71d8d2d6408d:40957 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T03:55:09,707 DEBUG [71d8d2d6408d:40957 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T03:55:09,707 DEBUG [71d8d2d6408d:40957 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T03:55:09,707 DEBUG [71d8d2d6408d:40957 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T03:55:09,707 INFO [71d8d2d6408d:40957 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T03:55:09,707 INFO [71d8d2d6408d:40957 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T03:55:09,707 DEBUG [71d8d2d6408d:40957 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T03:55:09,714 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=71d8d2d6408d,36947,1732420508038 2024-11-24T03:55:09,718 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 71d8d2d6408d,36947,1732420508038, state=OPENING 2024-11-24T03:55:09,746 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-24T03:55:09,755 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36947-0x1016c44a74a0002, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T03:55:09,755 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40957-0x1016c44a74a0000, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T03:55:09,755 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46457-0x1016c44a74a0001, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T03:55:09,757 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T03:55:09,757 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T03:55:09,757 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T03:55:09,760 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T03:55:09,762 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=71d8d2d6408d,36947,1732420508038}] 2024-11-24T03:55:09,942 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T03:55:09,945 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57125, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T03:55:09,953 INFO [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-24T03:55:09,953 INFO [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T03:55:09,954 INFO [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-24T03:55:09,956 INFO [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=71d8d2d6408d%2C36947%2C1732420508038.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/WALs/71d8d2d6408d,36947,1732420508038, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/oldWALs, maxLogs=55 2024-11-24T03:55:09,975 INFO [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/WALs/71d8d2d6408d,36947,1732420508038/71d8d2d6408d%2C36947%2C1732420508038.meta.1732420509958.meta 2024-11-24T03:55:09,975 DEBUG [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T03:55:09,976 DEBUG [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-24T03:55:09,978 DEBUG [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-24T03:55:09,980 DEBUG [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-24T03:55:09,984 INFO [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-24T03:55:09,988 DEBUG [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-24T03:55:09,988 DEBUG [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T03:55:09,989 DEBUG [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-24T03:55:09,989 DEBUG [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-24T03:55:09,990 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-24T03:55:09,991 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T03:55:09,991 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T03:55:09,991 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T03:55:09,991 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-24T03:55:09,992 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T03:55:09,992 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T03:55:09,992 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T03:55:09,993 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-24T03:55:09,993 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T03:55:09,993 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T03:55:09,993 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T03:55:09,994 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-24T03:55:09,994 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T03:55:09,995 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T03:55:09,995 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T03:55:09,995 DEBUG [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T03:55:09,996 DEBUG [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740 2024-11-24T03:55:09,997 DEBUG [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740 2024-11-24T03:55:09,998 DEBUG [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T03:55:09,998 DEBUG [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T03:55:09,998 DEBUG [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T03:55:09,999 DEBUG [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T03:55:10,000 INFO [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73843832, jitterRate=0.10035884380340576}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T03:55:10,000 DEBUG [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-24T03:55:10,002 DEBUG [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732420509989Writing region info on filesystem at 1732420509989Initializing all the Stores at 1732420509990 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732420509990Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732420509990Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732420509990Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732420509990Cleaning up temporary data from old regions at 1732420509998 (+8 ms)Running coprocessor post-open hooks at 1732420510000 (+2 ms)Region opened successfully at 1732420510002 (+2 ms) 2024-11-24T03:55:10,008 INFO [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732420509932 2024-11-24T03:55:10,018 DEBUG [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-24T03:55:10,019 INFO [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-24T03:55:10,020 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=71d8d2d6408d,36947,1732420508038 2024-11-24T03:55:10,021 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 71d8d2d6408d,36947,1732420508038, state=OPEN 2024-11-24T03:55:10,054 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40957-0x1016c44a74a0000, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T03:55:10,054 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46457-0x1016c44a74a0001, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T03:55:10,054 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36947-0x1016c44a74a0002, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T03:55:10,054 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T03:55:10,055 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T03:55:10,055 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T03:55:10,055 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=71d8d2d6408d,36947,1732420508038 2024-11-24T03:55:10,060 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-24T03:55:10,060 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=71d8d2d6408d,36947,1732420508038 in 293 msec 2024-11-24T03:55:10,065 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-24T03:55:10,065 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 789 msec 2024-11-24T03:55:10,066 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T03:55:10,066 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-24T03:55:10,086 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T03:55:10,088 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=71d8d2d6408d,36947,1732420508038, seqNum=-1] 2024-11-24T03:55:10,112 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T03:55:10,114 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58175, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T03:55:10,131 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0500 sec 2024-11-24T03:55:10,132 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732420510131, completionTime=-1 2024-11-24T03:55:10,134 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-24T03:55:10,134 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T03:55:10,157 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T03:55:10,157 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732420570157 2024-11-24T03:55:10,157 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732420630157 2024-11-24T03:55:10,157 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 22 msec 2024-11-24T03:55:10,159 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-24T03:55:10,165 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=71d8d2d6408d,40957,1732420507038-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:10,165 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=71d8d2d6408d,40957,1732420507038-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:10,165 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=71d8d2d6408d,40957,1732420507038-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:10,167 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-71d8d2d6408d:40957, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:10,167 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:10,167 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:10,173 DEBUG [master/71d8d2d6408d:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-24T03:55:10,197 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.015sec 2024-11-24T03:55:10,199 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-24T03:55:10,201 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-24T03:55:10,202 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-24T03:55:10,202 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-24T03:55:10,202 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-24T03:55:10,203 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=71d8d2d6408d,40957,1732420507038-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T03:55:10,204 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=71d8d2d6408d,40957,1732420507038-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-24T03:55:10,205 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-24T03:55:10,206 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-24T03:55:10,206 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=71d8d2d6408d,40957,1732420507038-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:10,213 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@218aa893, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T03:55:10,215 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-24T03:55:10,215 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-24T03:55:10,220 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 71d8d2d6408d,40957,-1 for getting cluster id 2024-11-24T03:55:10,223 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T03:55:10,231 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5ef32ff8-f46c-42b9-80e5-b7f373e9c523' 2024-11-24T03:55:10,233 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T03:55:10,234 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5ef32ff8-f46c-42b9-80e5-b7f373e9c523" 2024-11-24T03:55:10,236 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d3ee9a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T03:55:10,236 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [71d8d2d6408d,40957,-1] 2024-11-24T03:55:10,238 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T03:55:10,240 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T03:55:10,241 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37628, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T03:55:10,245 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f82a59d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T03:55:10,246 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T03:55:10,254 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=71d8d2d6408d,36947,1732420508038, seqNum=-1] 2024-11-24T03:55:10,254 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T03:55:10,257 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60688, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T03:55:10,287 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=71d8d2d6408d,40957,1732420507038 2024-11-24T03:55:10,291 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T03:55:10,296 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 71d8d2d6408d,40957,1732420507038 2024-11-24T03:55:10,299 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@61746f74 2024-11-24T03:55:10,303 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T03:55:10,307 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37634, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T03:55:10,312 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40957 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1732420510287', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T03:55:10,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40957 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732420510287 2024-11-24T03:55:10,320 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732420510287 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T03:55:10,323 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40957 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1732420510287" procId is: 4 2024-11-24T03:55:10,323 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T03:55:10,325 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732420510287 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T03:55:10,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T03:55:10,339 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732420510287-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 36e65fc6af1c020e7df60d5545722c58, NAME => 'testBlockEvictionOnRegionMove1732420510287,,1732420510309.36e65fc6af1c020e7df60d5545722c58.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1732420510287', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b 2024-11-24T03:55:10,346 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732420510287-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732420510287,,1732420510309.36e65fc6af1c020e7df60d5545722c58.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T03:55:10,346 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732420510287-pool-0 {}] regionserver.HRegion(1722): Closing 36e65fc6af1c020e7df60d5545722c58, disabling compactions & flushes 2024-11-24T03:55:10,346 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732420510287-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732420510287,,1732420510309.36e65fc6af1c020e7df60d5545722c58. 2024-11-24T03:55:10,347 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732420510287-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732420510287,,1732420510309.36e65fc6af1c020e7df60d5545722c58. 2024-11-24T03:55:10,347 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732420510287-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732420510287,,1732420510309.36e65fc6af1c020e7df60d5545722c58. after waiting 0 ms 2024-11-24T03:55:10,347 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732420510287-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732420510287,,1732420510309.36e65fc6af1c020e7df60d5545722c58. 2024-11-24T03:55:10,347 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732420510287-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732420510287,,1732420510309.36e65fc6af1c020e7df60d5545722c58. 2024-11-24T03:55:10,347 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732420510287-pool-0 {}] regionserver.HRegion(1676): Region close journal for 36e65fc6af1c020e7df60d5545722c58: Waiting for close lock at 1732420510346Disabling compacts and flushes for region at 1732420510346Disabling writes for close at 1732420510347 (+1 ms)Writing region close event to WAL at 1732420510347Closed at 1732420510347 2024-11-24T03:55:10,349 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732420510287 execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T03:55:10,353 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1732420510287,,1732420510309.36e65fc6af1c020e7df60d5545722c58.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732420510349"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732420510349"}]},"ts":"1732420510349"} 2024-11-24T03:55:10,358 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T03:55:10,360 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732420510287 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T03:55:10,362 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732420510287","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732420510360"}]},"ts":"1732420510360"} 2024-11-24T03:55:10,367 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732420510287, state=ENABLING in hbase:meta 2024-11-24T03:55:10,368 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {71d8d2d6408d=0} racks are {/default-rack=0} 2024-11-24T03:55:10,368 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T03:55:10,368 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T03:55:10,368 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T03:55:10,368 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T03:55:10,368 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T03:55:10,369 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T03:55:10,369 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T03:55:10,370 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732420510287, region=36e65fc6af1c020e7df60d5545722c58, ASSIGN}] 2024-11-24T03:55:10,372 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732420510287, region=36e65fc6af1c020e7df60d5545722c58, ASSIGN 2024-11-24T03:55:10,373 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732420510287, region=36e65fc6af1c020e7df60d5545722c58, ASSIGN; state=OFFLINE, location=71d8d2d6408d,46457,1732420507885; forceNewPlan=false, retain=false 2024-11-24T03:55:10,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T03:55:10,527 INFO [71d8d2d6408d:40957 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T03:55:10,529 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=36e65fc6af1c020e7df60d5545722c58, regionState=OPENING, regionLocation=71d8d2d6408d,46457,1732420507885 2024-11-24T03:55:10,533 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732420510287, region=36e65fc6af1c020e7df60d5545722c58, ASSIGN because future has completed 2024-11-24T03:55:10,535 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 36e65fc6af1c020e7df60d5545722c58, server=71d8d2d6408d,46457,1732420507885}] 2024-11-24T03:55:10,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T03:55:10,689 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T03:55:10,693 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37807, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T03:55:10,699 INFO [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732420510287,,1732420510309.36e65fc6af1c020e7df60d5545722c58. 2024-11-24T03:55:10,700 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 36e65fc6af1c020e7df60d5545722c58, NAME => 'testBlockEvictionOnRegionMove1732420510287,,1732420510309.36e65fc6af1c020e7df60d5545722c58.', STARTKEY => '', ENDKEY => ''} 2024-11-24T03:55:10,701 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732420510287 36e65fc6af1c020e7df60d5545722c58 2024-11-24T03:55:10,702 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732420510287,,1732420510309.36e65fc6af1c020e7df60d5545722c58.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T03:55:10,702 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 36e65fc6af1c020e7df60d5545722c58 2024-11-24T03:55:10,702 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 36e65fc6af1c020e7df60d5545722c58 2024-11-24T03:55:10,704 INFO [StoreOpener-36e65fc6af1c020e7df60d5545722c58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 36e65fc6af1c020e7df60d5545722c58 2024-11-24T03:55:10,708 INFO [StoreOpener-36e65fc6af1c020e7df60d5545722c58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 36e65fc6af1c020e7df60d5545722c58 columnFamilyName family 2024-11-24T03:55:10,708 DEBUG [StoreOpener-36e65fc6af1c020e7df60d5545722c58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T03:55:10,709 INFO [StoreOpener-36e65fc6af1c020e7df60d5545722c58-1 {}] regionserver.HStore(327): Store=36e65fc6af1c020e7df60d5545722c58/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T03:55:10,709 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 36e65fc6af1c020e7df60d5545722c58 2024-11-24T03:55:10,711 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/default/testBlockEvictionOnRegionMove1732420510287/36e65fc6af1c020e7df60d5545722c58 2024-11-24T03:55:10,711 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/default/testBlockEvictionOnRegionMove1732420510287/36e65fc6af1c020e7df60d5545722c58 2024-11-24T03:55:10,711 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 36e65fc6af1c020e7df60d5545722c58 2024-11-24T03:55:10,711 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 36e65fc6af1c020e7df60d5545722c58 2024-11-24T03:55:10,712 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 36e65fc6af1c020e7df60d5545722c58 2024-11-24T03:55:10,720 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/default/testBlockEvictionOnRegionMove1732420510287/36e65fc6af1c020e7df60d5545722c58/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T03:55:10,720 INFO [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 36e65fc6af1c020e7df60d5545722c58; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61316800, jitterRate=-0.08630847930908203}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T03:55:10,721 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 36e65fc6af1c020e7df60d5545722c58 2024-11-24T03:55:10,722 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 36e65fc6af1c020e7df60d5545722c58: Running coprocessor pre-open hook at 1732420510703Writing region info on filesystem at 1732420510703Initializing all the Stores at 1732420510703Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732420510703Cleaning up temporary data from old regions at 1732420510711 (+8 ms)Running coprocessor post-open hooks at 1732420510721 (+10 ms)Region opened successfully at 1732420510722 (+1 ms) 2024-11-24T03:55:10,723 INFO [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732420510287,,1732420510309.36e65fc6af1c020e7df60d5545722c58., pid=6, masterSystemTime=1732420510688 2024-11-24T03:55:10,727 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732420510287,,1732420510309.36e65fc6af1c020e7df60d5545722c58. 2024-11-24T03:55:10,727 INFO [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732420510287,,1732420510309.36e65fc6af1c020e7df60d5545722c58. 2024-11-24T03:55:10,727 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=36e65fc6af1c020e7df60d5545722c58, regionState=OPEN, openSeqNum=2, regionLocation=71d8d2d6408d,46457,1732420507885 2024-11-24T03:55:10,731 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 36e65fc6af1c020e7df60d5545722c58, server=71d8d2d6408d,46457,1732420507885 because future has completed 2024-11-24T03:55:10,735 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-24T03:55:10,736 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 36e65fc6af1c020e7df60d5545722c58, server=71d8d2d6408d,46457,1732420507885 in 198 msec 2024-11-24T03:55:10,737 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-24T03:55:10,738 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732420510287, region=36e65fc6af1c020e7df60d5545722c58, ASSIGN in 365 msec 2024-11-24T03:55:10,739 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732420510287 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T03:55:10,739 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732420510287","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732420510739"}]},"ts":"1732420510739"} 2024-11-24T03:55:10,742 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732420510287, state=ENABLED in hbase:meta 2024-11-24T03:55:10,743 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732420510287 execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T03:55:10,745 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732420510287 in 428 msec 2024-11-24T03:55:10,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40957 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T03:55:10,961 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1732420510287 get assigned. Timeout = 60000ms 2024-11-24T03:55:10,961 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1732420510287 completed 2024-11-24T03:55:10,964 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T03:55:10,969 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1732420510287 assigned to meta. Checking AM states. 2024-11-24T03:55:10,970 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T03:55:10,970 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1732420510287 assigned. 2024-11-24T03:55:10,978 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1732420510287', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1732420510287,,1732420510309.36e65fc6af1c020e7df60d5545722c58., hostname=71d8d2d6408d,46457,1732420507885, seqNum=2] 2024-11-24T03:55:10,979 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T03:55:10,981 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40272, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T03:55:10,991 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 36e65fc6af1c020e7df60d5545722c58 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-24T03:55:11,012 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on be55ded90caa44ce9cda70000eaaeaa6 2024-11-24T03:55:11,046 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/default/testBlockEvictionOnRegionMove1732420510287/36e65fc6af1c020e7df60d5545722c58/.tmp/family/be55ded90caa44ce9cda70000eaaeaa6 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-24T03:55:11,051 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/default/testBlockEvictionOnRegionMove1732420510287/36e65fc6af1c020e7df60d5545722c58/.tmp/family/be55ded90caa44ce9cda70000eaaeaa6 2024-11-24T03:55:11,081 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/default/testBlockEvictionOnRegionMove1732420510287/36e65fc6af1c020e7df60d5545722c58/.tmp/family/be55ded90caa44ce9cda70000eaaeaa6 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/default/testBlockEvictionOnRegionMove1732420510287/36e65fc6af1c020e7df60d5545722c58/family/be55ded90caa44ce9cda70000eaaeaa6 2024-11-24T03:55:11,082 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/default/testBlockEvictionOnRegionMove1732420510287/36e65fc6af1c020e7df60d5545722c58/family/be55ded90caa44ce9cda70000eaaeaa6, delay=920 ms 2024-11-24T03:55:11,084 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/default/testBlockEvictionOnRegionMove1732420510287/36e65fc6af1c020e7df60d5545722c58/family/be55ded90caa44ce9cda70000eaaeaa6, entries=2, sequenceid=6, filesize=4.9 K 2024-11-24T03:55:11,089 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 36e65fc6af1c020e7df60d5545722c58 in 99ms, sequenceid=6, compaction requested=false 2024-11-24T03:55:11,090 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1732420510287' 2024-11-24T03:55:11,092 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 36e65fc6af1c020e7df60d5545722c58: 2024-11-24T03:55:12,005 DEBUG [hfile-prefetch-1732420511082 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key be55ded90caa44ce9cda70000eaaeaa6_0. 2024-11-24T03:55:12,011 DEBUG [hfile-prefetch-1732420511082 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key be55ded90caa44ce9cda70000eaaeaa6_117. 2024-11-24T03:55:12,012 DEBUG [hfile-prefetch-1732420511082 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/default/testBlockEvictionOnRegionMove1732420510287/36e65fc6af1c020e7df60d5545722c58/family/be55ded90caa44ce9cda70000eaaeaa6, with total blocks 2, and data blocks 1 2024-11-24T03:55:12,013 DEBUG [hfile-prefetch-1732420511082 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-24T03:55:12,013 DEBUG [hfile-prefetch-1732420511082 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/default/testBlockEvictionOnRegionMove1732420510287/36e65fc6af1c020e7df60d5545722c58/family/be55ded90caa44ce9cda70000eaaeaa6, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-24T03:55:12,013 DEBUG [hfile-prefetch-1732420511082 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/default/testBlockEvictionOnRegionMove1732420510287/36e65fc6af1c020e7df60d5545722c58/family/be55ded90caa44ce9cda70000eaaeaa6. Will sleep 100ms and try the verification again. 2024-11-24T03:55:12,106 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40957 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=36e65fc6af1c020e7df60d5545722c58, source=71d8d2d6408d,46457,1732420507885, destination=71d8d2d6408d,36947,1732420508038, warming up region on 71d8d2d6408d,36947,1732420508038 2024-11-24T03:55:12,108 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40957 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=36e65fc6af1c020e7df60d5545722c58, source=71d8d2d6408d,46457,1732420507885, destination=71d8d2d6408d,36947,1732420508038, running balancer 2024-11-24T03:55:12,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40957 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732420510287, region=36e65fc6af1c020e7df60d5545722c58, REOPEN/MOVE 2024-11-24T03:55:12,110 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732420510287, region=36e65fc6af1c020e7df60d5545722c58, REOPEN/MOVE 2024-11-24T03:55:12,111 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36947 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1732420510287,,1732420510309.36e65fc6af1c020e7df60d5545722c58. 2024-11-24T03:55:12,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36947 {}] regionserver.HRegion(7855): Warmup {ENCODED => 36e65fc6af1c020e7df60d5545722c58, NAME => 'testBlockEvictionOnRegionMove1732420510287,,1732420510309.36e65fc6af1c020e7df60d5545722c58.', STARTKEY => '', ENDKEY => ''} 2024-11-24T03:55:12,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36947 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732420510287,,1732420510309.36e65fc6af1c020e7df60d5545722c58.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T03:55:12,112 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=36e65fc6af1c020e7df60d5545722c58, regionState=CLOSING, regionLocation=71d8d2d6408d,46457,1732420507885 2024-11-24T03:55:12,114 DEBUG [hfile-prefetch-1732420511082 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/default/testBlockEvictionOnRegionMove1732420510287/36e65fc6af1c020e7df60d5545722c58/family/be55ded90caa44ce9cda70000eaaeaa6, with total blocks 2, and data blocks 1 2024-11-24T03:55:12,114 DEBUG [hfile-prefetch-1732420511082 {}] bucket.BucketCache(2285): iterating over 1 entries in the backing map 2024-11-24T03:55:12,114 INFO [StoreOpener-36e65fc6af1c020e7df60d5545722c58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 36e65fc6af1c020e7df60d5545722c58 2024-11-24T03:55:12,114 DEBUG [hfile-prefetch-1732420511082 {}] bucket.BucketCache(2293): found block for file be55ded90caa44ce9cda70000eaaeaa6 in the backing map. Acquiring read lock for offset 0 2024-11-24T03:55:12,114 DEBUG [hfile-prefetch-1732420511082 {}] bucket.BucketCache(2305): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/default/testBlockEvictionOnRegionMove1732420510287/36e65fc6af1c020e7df60d5545722c58/family/be55ded90caa44ce9cda70000eaaeaa6 has now been fully cached. 2024-11-24T03:55:12,114 DEBUG [hfile-prefetch-1732420511082 {}] hfile.PrefetchExecutor(121): Prefetch completed for be55ded90caa44ce9cda70000eaaeaa6 2024-11-24T03:55:12,115 INFO [StoreOpener-36e65fc6af1c020e7df60d5545722c58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 36e65fc6af1c020e7df60d5545722c58 columnFamilyName family 2024-11-24T03:55:12,115 DEBUG [StoreOpener-36e65fc6af1c020e7df60d5545722c58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T03:55:12,115 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732420510287, region=36e65fc6af1c020e7df60d5545722c58, REOPEN/MOVE because future has completed 2024-11-24T03:55:12,116 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40957 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=71d8d2d6408d,46457,1732420507885, table=testBlockEvictionOnRegionMove1732420510287, region=36e65fc6af1c020e7df60d5545722c58. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T03:55:12,116 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-24T03:55:12,116 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 36e65fc6af1c020e7df60d5545722c58, server=71d8d2d6408d,46457,1732420507885}] 2024-11-24T03:55:12,119 DEBUG [StoreOpener-36e65fc6af1c020e7df60d5545722c58-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/default/testBlockEvictionOnRegionMove1732420510287/36e65fc6af1c020e7df60d5545722c58/family/be55ded90caa44ce9cda70000eaaeaa6 2024-11-24T03:55:12,120 INFO [StoreOpener-36e65fc6af1c020e7df60d5545722c58-1 {}] regionserver.HStore(327): Store=36e65fc6af1c020e7df60d5545722c58/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T03:55:12,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36947 {}] regionserver.HRegion(1722): Closing 36e65fc6af1c020e7df60d5545722c58, disabling compactions & flushes 2024-11-24T03:55:12,120 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36947 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732420510287,,1732420510309.36e65fc6af1c020e7df60d5545722c58. 2024-11-24T03:55:12,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36947 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732420510287,,1732420510309.36e65fc6af1c020e7df60d5545722c58. 2024-11-24T03:55:12,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36947 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732420510287,,1732420510309.36e65fc6af1c020e7df60d5545722c58. after waiting 0 ms 2024-11-24T03:55:12,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36947 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732420510287,,1732420510309.36e65fc6af1c020e7df60d5545722c58. 2024-11-24T03:55:12,122 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36947 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732420510287,,1732420510309.36e65fc6af1c020e7df60d5545722c58. 2024-11-24T03:55:12,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36947 {}] regionserver.HRegion(1676): Region close journal for 36e65fc6af1c020e7df60d5545722c58: Waiting for close lock at 1732420512120Disabling compacts and flushes for region at 1732420512120Disabling writes for close at 1732420512120Writing region close event to WAL at 1732420512122 (+2 ms)Closed at 1732420512122 2024-11-24T03:55:12,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40957 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-24T03:55:12,188 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-24T03:55:12,188 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-24T03:55:12,189 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-24T03:55:12,189 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T03:55:12,189 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-24T03:55:12,189 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-11-24T03:55:12,189 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T03:55:12,189 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T03:55:12,210 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-24T03:55:12,210 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732420512189 2024-11-24T03:55:12,210 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-24T03:55:12,213 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-24T03:55:12,213 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1732420512188 2024-11-24T03:55:12,213 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-24T03:55:12,277 INFO [RS_CLOSE_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 36e65fc6af1c020e7df60d5545722c58 2024-11-24T03:55:12,278 DEBUG [RS_CLOSE_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-24T03:55:12,279 DEBUG [RS_CLOSE_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 36e65fc6af1c020e7df60d5545722c58, disabling compactions & flushes 2024-11-24T03:55:12,279 INFO [RS_CLOSE_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732420510287,,1732420510309.36e65fc6af1c020e7df60d5545722c58. 2024-11-24T03:55:12,279 DEBUG [RS_CLOSE_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732420510287,,1732420510309.36e65fc6af1c020e7df60d5545722c58. 2024-11-24T03:55:12,280 DEBUG [RS_CLOSE_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732420510287,,1732420510309.36e65fc6af1c020e7df60d5545722c58. after waiting 0 ms 2024-11-24T03:55:12,280 DEBUG [RS_CLOSE_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732420510287,,1732420510309.36e65fc6af1c020e7df60d5545722c58. 2024-11-24T03:55:12,283 DEBUG [StoreFileCloser-36e65fc6af1c020e7df60d5545722c58-family-1 {}] bucket.BucketCache(768): Removing file be55ded90caa44ce9cda70000eaaeaa6 for region 36e65fc6af1c020e7df60d5545722c58 2024-11-24T03:55:12,284 DEBUG [StoreFileCloser-36e65fc6af1c020e7df60d5545722c58-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file be55ded90caa44ce9cda70000eaaeaa6, starting offset: 0, end offset: 9223372036854775807 2024-11-24T03:55:12,287 DEBUG [StoreFileCloser-36e65fc6af1c020e7df60d5545722c58-family-1 {}] bucket.BucketCache(856): removed key be55ded90caa44ce9cda70000eaaeaa6_0 from back map with offset lock 2094080 in the evict process 2024-11-24T03:55:12,292 DEBUG [RS_CLOSE_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/default/testBlockEvictionOnRegionMove1732420510287/36e65fc6af1c020e7df60d5545722c58/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-24T03:55:12,293 INFO [RS_CLOSE_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732420510287,,1732420510309.36e65fc6af1c020e7df60d5545722c58. 2024-11-24T03:55:12,293 DEBUG [RS_CLOSE_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 36e65fc6af1c020e7df60d5545722c58: Waiting for close lock at 1732420512279Running coprocessor pre-close hooks at 1732420512279Disabling compacts and flushes for region at 1732420512279Disabling writes for close at 1732420512280 (+1 ms)Writing region close event to WAL at 1732420512287 (+7 ms)Running coprocessor post-close hooks at 1732420512292 (+5 ms)Closed at 1732420512293 (+1 ms) 2024-11-24T03:55:12,294 INFO [RS_CLOSE_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 36e65fc6af1c020e7df60d5545722c58 move to 71d8d2d6408d,36947,1732420508038 record at close sequenceid=6 2024-11-24T03:55:12,298 INFO [RS_CLOSE_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 36e65fc6af1c020e7df60d5545722c58 2024-11-24T03:55:12,298 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=36e65fc6af1c020e7df60d5545722c58, regionState=CLOSED 2024-11-24T03:55:12,301 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 36e65fc6af1c020e7df60d5545722c58, server=71d8d2d6408d,46457,1732420507885 because future has completed 2024-11-24T03:55:12,304 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-24T03:55:12,304 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 36e65fc6af1c020e7df60d5545722c58, server=71d8d2d6408d,46457,1732420507885 in 186 msec 2024-11-24T03:55:12,305 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732420510287, region=36e65fc6af1c020e7df60d5545722c58, REOPEN/MOVE; state=CLOSED, location=71d8d2d6408d,36947,1732420508038; forceNewPlan=false, retain=false 2024-11-24T03:55:12,310 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-24T03:55:12,310 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-24T03:55:12,310 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T03:55:12,311 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T03:55:12,314 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-24T03:55:12,314 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732420512310 2024-11-24T03:55:12,314 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-24T03:55:12,456 INFO [71d8d2d6408d:40957 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T03:55:12,457 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=36e65fc6af1c020e7df60d5545722c58, regionState=OPENING, regionLocation=71d8d2d6408d,36947,1732420508038 2024-11-24T03:55:12,464 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732420510287, region=36e65fc6af1c020e7df60d5545722c58, REOPEN/MOVE because future has completed 2024-11-24T03:55:12,466 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 36e65fc6af1c020e7df60d5545722c58, server=71d8d2d6408d,36947,1732420508038}] 2024-11-24T03:55:12,628 INFO [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732420510287,,1732420510309.36e65fc6af1c020e7df60d5545722c58. 2024-11-24T03:55:12,629 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 36e65fc6af1c020e7df60d5545722c58, NAME => 'testBlockEvictionOnRegionMove1732420510287,,1732420510309.36e65fc6af1c020e7df60d5545722c58.', STARTKEY => '', ENDKEY => ''} 2024-11-24T03:55:12,630 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732420510287 36e65fc6af1c020e7df60d5545722c58 2024-11-24T03:55:12,630 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732420510287,,1732420510309.36e65fc6af1c020e7df60d5545722c58.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T03:55:12,631 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 36e65fc6af1c020e7df60d5545722c58 2024-11-24T03:55:12,631 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 36e65fc6af1c020e7df60d5545722c58 2024-11-24T03:55:12,632 INFO [StoreOpener-36e65fc6af1c020e7df60d5545722c58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 36e65fc6af1c020e7df60d5545722c58 2024-11-24T03:55:12,633 INFO [StoreOpener-36e65fc6af1c020e7df60d5545722c58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 36e65fc6af1c020e7df60d5545722c58 columnFamilyName family 2024-11-24T03:55:12,633 DEBUG [StoreOpener-36e65fc6af1c020e7df60d5545722c58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T03:55:12,635 DEBUG [StoreFileOpener-36e65fc6af1c020e7df60d5545722c58-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/default/testBlockEvictionOnRegionMove1732420510287/36e65fc6af1c020e7df60d5545722c58/family/be55ded90caa44ce9cda70000eaaeaa6, delay=940 ms 2024-11-24T03:55:12,636 DEBUG [StoreOpener-36e65fc6af1c020e7df60d5545722c58-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/default/testBlockEvictionOnRegionMove1732420510287/36e65fc6af1c020e7df60d5545722c58/family/be55ded90caa44ce9cda70000eaaeaa6 2024-11-24T03:55:12,636 INFO [StoreOpener-36e65fc6af1c020e7df60d5545722c58-1 {}] regionserver.HStore(327): Store=36e65fc6af1c020e7df60d5545722c58/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T03:55:12,636 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 36e65fc6af1c020e7df60d5545722c58 2024-11-24T03:55:12,637 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/default/testBlockEvictionOnRegionMove1732420510287/36e65fc6af1c020e7df60d5545722c58 2024-11-24T03:55:12,637 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/default/testBlockEvictionOnRegionMove1732420510287/36e65fc6af1c020e7df60d5545722c58 2024-11-24T03:55:12,637 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 36e65fc6af1c020e7df60d5545722c58 2024-11-24T03:55:12,638 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 36e65fc6af1c020e7df60d5545722c58 2024-11-24T03:55:12,641 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 36e65fc6af1c020e7df60d5545722c58 2024-11-24T03:55:12,642 INFO [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 36e65fc6af1c020e7df60d5545722c58; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60075211, jitterRate=-0.10480959713459015}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T03:55:12,642 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 36e65fc6af1c020e7df60d5545722c58 2024-11-24T03:55:12,643 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 36e65fc6af1c020e7df60d5545722c58: Running coprocessor pre-open hook at 1732420512631Writing region info on filesystem at 1732420512631Initializing all the Stores at 1732420512631Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732420512631Cleaning up temporary data from old regions at 1732420512638 (+7 ms)Running coprocessor post-open hooks at 1732420512642 (+4 ms)Region opened successfully at 1732420512643 (+1 ms) 2024-11-24T03:55:12,644 INFO [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732420510287,,1732420510309.36e65fc6af1c020e7df60d5545722c58., pid=9, masterSystemTime=1732420512620 2024-11-24T03:55:12,647 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732420510287,,1732420510309.36e65fc6af1c020e7df60d5545722c58. 2024-11-24T03:55:12,647 INFO [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732420510287,,1732420510309.36e65fc6af1c020e7df60d5545722c58. 2024-11-24T03:55:12,647 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=36e65fc6af1c020e7df60d5545722c58, regionState=OPEN, openSeqNum=10, regionLocation=71d8d2d6408d,36947,1732420508038 2024-11-24T03:55:12,650 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 36e65fc6af1c020e7df60d5545722c58, server=71d8d2d6408d,36947,1732420508038 because future has completed 2024-11-24T03:55:12,653 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-24T03:55:12,654 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 36e65fc6af1c020e7df60d5545722c58, server=71d8d2d6408d,36947,1732420508038 in 186 msec 2024-11-24T03:55:12,655 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732420510287, region=36e65fc6af1c020e7df60d5545722c58, REOPEN/MOVE in 545 msec 2024-11-24T03:55:12,663 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-24T03:55:12,663 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T03:55:12,664 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T03:55:12,668 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T03:55:12,669 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T03:55:12,669 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T03:55:12,669 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-24T03:55:12,669 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1694793401, stopped=false 2024-11-24T03:55:12,670 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=71d8d2d6408d,40957,1732420507038 2024-11-24T03:55:12,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40957-0x1016c44a74a0000, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T03:55:12,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36947-0x1016c44a74a0002, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T03:55:12,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46457-0x1016c44a74a0001, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T03:55:12,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40957-0x1016c44a74a0000, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T03:55:12,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46457-0x1016c44a74a0001, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T03:55:12,705 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T03:55:12,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36947-0x1016c44a74a0002, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T03:55:12,706 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T03:55:12,707 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36947-0x1016c44a74a0002, quorum=127.0.0.1:59166, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T03:55:12,707 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T03:55:12,707 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46457-0x1016c44a74a0001, quorum=127.0.0.1:59166, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T03:55:12,707 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T03:55:12,707 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40957-0x1016c44a74a0000, quorum=127.0.0.1:59166, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T03:55:12,708 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '71d8d2d6408d,46457,1732420507885' ***** 2024-11-24T03:55:12,708 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T03:55:12,708 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '71d8d2d6408d,36947,1732420508038' ***** 2024-11-24T03:55:12,708 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T03:55:12,709 INFO [RS:0;71d8d2d6408d:46457 {}] bucket.BucketCache(1738): Disabling cache 2024-11-24T03:55:12,709 INFO [RS:1;71d8d2d6408d:36947 {}] bucket.BucketCache(1738): Disabling cache 2024-11-24T03:55:12,709 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-24T03:55:12,709 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-24T03:55:12,709 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-24T03:55:12,710 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-24T03:55:12,710 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-24T03:55:12,710 INFO [RS:0;71d8d2d6408d:46457 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/bucket.persistence 2024-11-24T03:55:12,710 INFO [RS:1;71d8d2d6408d:36947 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/bucket.persistence 2024-11-24T03:55:12,710 INFO [RS:0;71d8d2d6408d:46457 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-24T03:55:12,710 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-24T03:55:12,710 INFO [RS:1;71d8d2d6408d:36947 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-24T03:55:12,710 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-24T03:55:12,710 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-24T03:55:12,720 DEBUG [RS:0;71d8d2d6408d:46457 {}] bucket.BucketCache(1487): Thread RS:0;71d8d2d6408d:46457 started persisting bucket cache to file 2024-11-24T03:55:12,720 DEBUG [RS:0;71d8d2d6408d:46457 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T03:55:12,720 DEBUG [RS:1;71d8d2d6408d:36947 {}] bucket.BucketCache(1487): Thread RS:1;71d8d2d6408d:36947 started persisting bucket cache to file 2024-11-24T03:55:12,720 DEBUG [RS:0;71d8d2d6408d:46457 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T03:55:12,720 DEBUG [RS:1;71d8d2d6408d:36947 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T03:55:12,720 DEBUG [RS:1;71d8d2d6408d:36947 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T03:55:12,723 DEBUG [RS:1;71d8d2d6408d:36947 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-24T03:55:12,723 DEBUG [RS:0;71d8d2d6408d:46457 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-24T03:55:12,723 DEBUG [RS:1;71d8d2d6408d:36947 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732420512720 2024-11-24T03:55:12,723 DEBUG [RS:0;71d8d2d6408d:46457 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732420512720 2024-11-24T03:55:12,723 DEBUG [RS:1;71d8d2d6408d:36947 {}] bucket.BucketCache(1509): Thread RS:1;71d8d2d6408d:36947 finished persisting bucket cache to file, renaming 2024-11-24T03:55:12,723 DEBUG [RS:0;71d8d2d6408d:46457 {}] bucket.BucketCache(1509): Thread RS:0;71d8d2d6408d:46457 finished persisting bucket cache to file, renaming 2024-11-24T03:55:12,723 WARN [RS:0;71d8d2d6408d:46457 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-24T03:55:12,723 INFO [RS:1;71d8d2d6408d:36947 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T03:55:12,723 INFO [RS:0;71d8d2d6408d:46457 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T03:55:12,724 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T03:55:12,724 INFO [RS:1;71d8d2d6408d:36947 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T03:55:12,724 INFO [RS:0;71d8d2d6408d:46457 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T03:55:12,724 INFO [RS:0;71d8d2d6408d:46457 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T03:55:12,724 INFO [RS:1;71d8d2d6408d:36947 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T03:55:12,724 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T03:55:12,724 INFO [RS:0;71d8d2d6408d:46457 {}] regionserver.HRegionServer(959): stopping server 71d8d2d6408d,46457,1732420507885 2024-11-24T03:55:12,724 INFO [RS:0;71d8d2d6408d:46457 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T03:55:12,724 INFO [RS:1;71d8d2d6408d:36947 {}] regionserver.HRegionServer(3091): Received CLOSE for 36e65fc6af1c020e7df60d5545722c58 2024-11-24T03:55:12,724 INFO [RS:0;71d8d2d6408d:46457 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;71d8d2d6408d:46457. 2024-11-24T03:55:12,724 DEBUG [RS:0;71d8d2d6408d:46457 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T03:55:12,724 DEBUG [RS:0;71d8d2d6408d:46457 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T03:55:12,724 INFO [RS:0;71d8d2d6408d:46457 {}] regionserver.HRegionServer(976): stopping server 71d8d2d6408d,46457,1732420507885; all regions closed. 2024-11-24T03:55:12,724 INFO [RS:1;71d8d2d6408d:36947 {}] regionserver.HRegionServer(959): stopping server 71d8d2d6408d,36947,1732420508038 2024-11-24T03:55:12,724 INFO [RS:1;71d8d2d6408d:36947 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T03:55:12,725 INFO [RS:1;71d8d2d6408d:36947 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;71d8d2d6408d:36947. 2024-11-24T03:55:12,725 DEBUG [RS:1;71d8d2d6408d:36947 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T03:55:12,725 DEBUG [RS:1;71d8d2d6408d:36947 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T03:55:12,725 INFO [RS:1;71d8d2d6408d:36947 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T03:55:12,725 INFO [RS:1;71d8d2d6408d:36947 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T03:55:12,725 INFO [RS:1;71d8d2d6408d:36947 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T03:55:12,725 INFO [RS_CLOSE_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1732420510287,,1732420510309.36e65fc6af1c020e7df60d5545722c58. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-24T03:55:12,725 INFO [RS:1;71d8d2d6408d:36947 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-24T03:55:12,725 DEBUG [RS_CLOSE_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 36e65fc6af1c020e7df60d5545722c58, disabling compactions & flushes 2024-11-24T03:55:12,725 INFO [RS_CLOSE_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732420510287,,1732420510309.36e65fc6af1c020e7df60d5545722c58. 2024-11-24T03:55:12,725 INFO [RS:1;71d8d2d6408d:36947 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-24T03:55:12,725 DEBUG [RS_CLOSE_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732420510287,,1732420510309.36e65fc6af1c020e7df60d5545722c58. 2024-11-24T03:55:12,726 DEBUG [RS:1;71d8d2d6408d:36947 {}] regionserver.HRegionServer(1325): Online Regions={36e65fc6af1c020e7df60d5545722c58=testBlockEvictionOnRegionMove1732420510287,,1732420510309.36e65fc6af1c020e7df60d5545722c58., 1588230740=hbase:meta,,1.1588230740} 2024-11-24T03:55:12,726 DEBUG [RS_CLOSE_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732420510287,,1732420510309.36e65fc6af1c020e7df60d5545722c58. after waiting 0 ms 2024-11-24T03:55:12,726 INFO [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-24T03:55:12,726 DEBUG [RS_CLOSE_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732420510287,,1732420510309.36e65fc6af1c020e7df60d5545722c58. 2024-11-24T03:55:12,726 DEBUG [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T03:55:12,726 INFO [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T03:55:12,726 DEBUG [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T03:55:12,726 DEBUG [RS:1;71d8d2d6408d:36947 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 36e65fc6af1c020e7df60d5545722c58 2024-11-24T03:55:12,726 DEBUG [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T03:55:12,726 DEBUG [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T03:55:12,726 DEBUG [StoreFileCloser-36e65fc6af1c020e7df60d5545722c58-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/default/testBlockEvictionOnRegionMove1732420510287/36e65fc6af1c020e7df60d5545722c58/family/be55ded90caa44ce9cda70000eaaeaa6 2024-11-24T03:55:12,726 INFO [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-24T03:55:12,729 DEBUG [RS:0;71d8d2d6408d:46457 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/oldWALs 2024-11-24T03:55:12,729 INFO [RS:0;71d8d2d6408d:46457 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 71d8d2d6408d%2C46457%2C1732420507885:(num 1732420509557) 2024-11-24T03:55:12,729 DEBUG [RS:0;71d8d2d6408d:46457 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T03:55:12,729 INFO [RS:0;71d8d2d6408d:46457 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T03:55:12,729 INFO [RS:0;71d8d2d6408d:46457 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T03:55:12,729 INFO [RS:0;71d8d2d6408d:46457 {}] hbase.ChoreService(370): Chore service for: regionserver/71d8d2d6408d:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T03:55:12,730 INFO [RS:0;71d8d2d6408d:46457 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T03:55:12,730 INFO [regionserver/71d8d2d6408d:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T03:55:12,730 INFO [RS:0;71d8d2d6408d:46457 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T03:55:12,730 INFO [RS:0;71d8d2d6408d:46457 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T03:55:12,730 INFO [RS:0;71d8d2d6408d:46457 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T03:55:12,730 INFO [RS:0;71d8d2d6408d:46457 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46457 2024-11-24T03:55:12,731 DEBUG [RS_CLOSE_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/default/testBlockEvictionOnRegionMove1732420510287/36e65fc6af1c020e7df60d5545722c58/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-24T03:55:12,732 INFO [RS_CLOSE_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732420510287,,1732420510309.36e65fc6af1c020e7df60d5545722c58. 2024-11-24T03:55:12,732 DEBUG [RS_CLOSE_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 36e65fc6af1c020e7df60d5545722c58: Waiting for close lock at 1732420512725Running coprocessor pre-close hooks at 1732420512725Disabling compacts and flushes for region at 1732420512725Disabling writes for close at 1732420512726 (+1 ms)Writing region close event to WAL at 1732420512727 (+1 ms)Running coprocessor post-close hooks at 1732420512731 (+4 ms)Closed at 1732420512732 (+1 ms) 2024-11-24T03:55:12,732 DEBUG [RS_CLOSE_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1732420510287,,1732420510309.36e65fc6af1c020e7df60d5545722c58. 2024-11-24T03:55:12,737 DEBUG [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ec389fbb76434e39826bd6aa6cd25d4c 2024-11-24T03:55:12,754 DEBUG [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740/.tmp/info/ec389fbb76434e39826bd6aa6cd25d4c is 205, key is testBlockEvictionOnRegionMove1732420510287,,1732420510309.36e65fc6af1c020e7df60d5545722c58./info:regioninfo/1732420512647/Put/seqid=0 2024-11-24T03:55:12,754 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46457-0x1016c44a74a0001, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/71d8d2d6408d,46457,1732420507885 2024-11-24T03:55:12,754 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40957-0x1016c44a74a0000, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T03:55:12,754 INFO [RS:0;71d8d2d6408d:46457 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T03:55:12,755 INFO [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740/.tmp/info/ec389fbb76434e39826bd6aa6cd25d4c 2024-11-24T03:55:12,755 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [71d8d2d6408d,46457,1732420507885] 2024-11-24T03:55:12,765 DEBUG [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ae9fc7c8397542148f3157dd1da88881 2024-11-24T03:55:12,770 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/71d8d2d6408d,46457,1732420507885 already deleted, retry=false 2024-11-24T03:55:12,771 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 71d8d2d6408d,46457,1732420507885 expired; onlineServers=1 2024-11-24T03:55:12,781 DEBUG [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740/.tmp/ns/ae9fc7c8397542148f3157dd1da88881 is 43, key is default/ns:d/1732420510119/Put/seqid=0 2024-11-24T03:55:12,782 INFO [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740/.tmp/ns/ae9fc7c8397542148f3157dd1da88881 2024-11-24T03:55:12,792 DEBUG [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d1341317b2bb4dcbb31d2b7c870dd847 2024-11-24T03:55:12,795 INFO [regionserver/71d8d2d6408d:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T03:55:12,795 INFO [regionserver/71d8d2d6408d:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T03:55:12,808 DEBUG [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740/.tmp/table/d1341317b2bb4dcbb31d2b7c870dd847 is 78, key is testBlockEvictionOnRegionMove1732420510287/table:state/1732420510739/Put/seqid=0 2024-11-24T03:55:12,809 INFO [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740/.tmp/table/d1341317b2bb4dcbb31d2b7c870dd847 2024-11-24T03:55:12,812 DEBUG [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740/.tmp/info/ec389fbb76434e39826bd6aa6cd25d4c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740/info/ec389fbb76434e39826bd6aa6cd25d4c 2024-11-24T03:55:12,813 DEBUG [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740/info/ec389fbb76434e39826bd6aa6cd25d4c, delay=981 ms 2024-11-24T03:55:12,814 INFO [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740/info/ec389fbb76434e39826bd6aa6cd25d4c, entries=15, sequenceid=15, filesize=7.6 K 2024-11-24T03:55:12,814 DEBUG [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740/.tmp/ns/ae9fc7c8397542148f3157dd1da88881 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740/ns/ae9fc7c8397542148f3157dd1da88881 2024-11-24T03:55:12,816 DEBUG [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740/ns/ae9fc7c8397542148f3157dd1da88881, delay=954 ms 2024-11-24T03:55:12,817 INFO [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740/ns/ae9fc7c8397542148f3157dd1da88881, entries=2, sequenceid=15, filesize=5.0 K 2024-11-24T03:55:12,818 DEBUG [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740/.tmp/table/d1341317b2bb4dcbb31d2b7c870dd847 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740/table/d1341317b2bb4dcbb31d2b7c870dd847 2024-11-24T03:55:12,819 DEBUG [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740/table/d1341317b2bb4dcbb31d2b7c870dd847, delay=990 ms 2024-11-24T03:55:12,821 INFO [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740/table/d1341317b2bb4dcbb31d2b7c870dd847, entries=2, sequenceid=15, filesize=5.3 K 2024-11-24T03:55:12,822 INFO [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 95ms, sequenceid=15, compaction requested=false 2024-11-24T03:55:12,822 DEBUG [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-24T03:55:12,823 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740/info/ec389fbb76434e39826bd6aa6cd25d4c 2024-11-24T03:55:12,824 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740/ns/ae9fc7c8397542148f3157dd1da88881 2024-11-24T03:55:12,825 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740/table/d1341317b2bb4dcbb31d2b7c870dd847 2024-11-24T03:55:12,829 DEBUG [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-24T03:55:12,829 DEBUG [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T03:55:12,829 INFO [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T03:55:12,830 DEBUG [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732420512725Running coprocessor pre-close hooks at 1732420512726 (+1 ms)Disabling compacts and flushes for region at 1732420512726Disabling writes for close at 1732420512726Obtaining lock to block concurrent updates at 1732420512726Preparing flush snapshotting stores in 1588230740 at 1732420512726Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1732420512727 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732420512727Flushing 1588230740/info: creating writer at 1732420512728 (+1 ms)Flushing 1588230740/info: appending metadata at 1732420512752 (+24 ms)Flushing 1588230740/info: closing flushed file at 1732420512752Flushing 1588230740/ns: creating writer at 1732420512759 (+7 ms)Flushing 1588230740/ns: appending metadata at 1732420512780 (+21 ms)Flushing 1588230740/ns: closing flushed file at 1732420512780Flushing 1588230740/table: creating writer at 1732420512786 (+6 ms)Flushing 1588230740/table: appending metadata at 1732420512807 (+21 ms)Flushing 1588230740/table: closing flushed file at 1732420512807Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23024ed1: reopening flushed file at 1732420512811 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f7c0d48: reopening flushed file at 1732420512814 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19431281: reopening flushed file at 1732420512817 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 95ms, sequenceid=15, compaction requested=false at 1732420512822 (+5 ms)Writing region close event to WAL at 1732420512825 (+3 ms)Running coprocessor post-close hooks at 1732420512829 (+4 ms)Closed at 1732420512829 2024-11-24T03:55:12,830 DEBUG [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-24T03:55:12,863 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46457-0x1016c44a74a0001, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T03:55:12,863 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46457-0x1016c44a74a0001, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T03:55:12,864 INFO [RS:0;71d8d2d6408d:46457 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T03:55:12,864 INFO [RS:0;71d8d2d6408d:46457 {}] regionserver.HRegionServer(1031): Exiting; stopping=71d8d2d6408d,46457,1732420507885; zookeeper connection closed. 2024-11-24T03:55:12,865 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@54a6a7ae {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@54a6a7ae 2024-11-24T03:55:12,926 INFO [RS:1;71d8d2d6408d:36947 {}] regionserver.HRegionServer(976): stopping server 71d8d2d6408d,36947,1732420508038; all regions closed. 2024-11-24T03:55:12,931 DEBUG [RS:1;71d8d2d6408d:36947 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/oldWALs 2024-11-24T03:55:12,931 INFO [RS:1;71d8d2d6408d:36947 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 71d8d2d6408d%2C36947%2C1732420508038.meta:.meta(num 1732420509958) 2024-11-24T03:55:12,935 DEBUG [RS:1;71d8d2d6408d:36947 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/oldWALs 2024-11-24T03:55:12,935 INFO [RS:1;71d8d2d6408d:36947 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 71d8d2d6408d%2C36947%2C1732420508038:(num 1732420509558) 2024-11-24T03:55:12,935 DEBUG [RS:1;71d8d2d6408d:36947 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T03:55:12,935 INFO [RS:1;71d8d2d6408d:36947 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T03:55:12,935 INFO [RS:1;71d8d2d6408d:36947 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T03:55:12,935 INFO [RS:1;71d8d2d6408d:36947 {}] hbase.ChoreService(370): Chore service for: regionserver/71d8d2d6408d:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T03:55:12,936 INFO [RS:1;71d8d2d6408d:36947 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T03:55:12,936 INFO [regionserver/71d8d2d6408d:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T03:55:12,936 INFO [RS:1;71d8d2d6408d:36947 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36947 2024-11-24T03:55:12,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40957-0x1016c44a74a0000, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T03:55:12,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36947-0x1016c44a74a0002, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/71d8d2d6408d,36947,1732420508038 2024-11-24T03:55:12,954 INFO [RS:1;71d8d2d6408d:36947 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T03:55:12,962 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [71d8d2d6408d,36947,1732420508038] 2024-11-24T03:55:12,970 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/71d8d2d6408d,36947,1732420508038 already deleted, retry=false 2024-11-24T03:55:12,971 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 71d8d2d6408d,36947,1732420508038 expired; onlineServers=0 2024-11-24T03:55:12,971 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '71d8d2d6408d,40957,1732420507038' ***** 2024-11-24T03:55:12,971 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-24T03:55:12,971 INFO [M:0;71d8d2d6408d:40957 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T03:55:12,971 INFO [M:0;71d8d2d6408d:40957 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T03:55:12,971 DEBUG [M:0;71d8d2d6408d:40957 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-24T03:55:12,971 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-24T03:55:12,971 DEBUG [M:0;71d8d2d6408d:40957 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-24T03:55:12,971 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster-HFileCleaner.large.0-1732420509189 {}] cleaner.HFileCleaner(306): Exit Thread[master/71d8d2d6408d:0:becomeActiveMaster-HFileCleaner.large.0-1732420509189,5,FailOnTimeoutGroup] 2024-11-24T03:55:12,971 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster-HFileCleaner.small.0-1732420509190 {}] cleaner.HFileCleaner(306): Exit Thread[master/71d8d2d6408d:0:becomeActiveMaster-HFileCleaner.small.0-1732420509190,5,FailOnTimeoutGroup] 2024-11-24T03:55:12,972 INFO [M:0;71d8d2d6408d:40957 {}] hbase.ChoreService(370): Chore service for: master/71d8d2d6408d:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-24T03:55:12,972 INFO [M:0;71d8d2d6408d:40957 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T03:55:12,972 DEBUG [M:0;71d8d2d6408d:40957 {}] master.HMaster(1795): Stopping service threads 2024-11-24T03:55:12,972 INFO [M:0;71d8d2d6408d:40957 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-24T03:55:12,972 INFO [M:0;71d8d2d6408d:40957 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T03:55:12,973 INFO [M:0;71d8d2d6408d:40957 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T03:55:12,973 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-24T03:55:12,979 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40957-0x1016c44a74a0000, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-24T03:55:12,979 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40957-0x1016c44a74a0000, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T03:55:12,979 DEBUG [M:0;71d8d2d6408d:40957 {}] zookeeper.ZKUtil(347): master:40957-0x1016c44a74a0000, quorum=127.0.0.1:59166, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-24T03:55:12,979 WARN [M:0;71d8d2d6408d:40957 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-24T03:55:12,980 INFO [M:0;71d8d2d6408d:40957 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/.lastflushedseqids 2024-11-24T03:55:12,990 INFO [M:0;71d8d2d6408d:40957 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T03:55:12,990 INFO [M:0;71d8d2d6408d:40957 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-24T03:55:12,990 DEBUG [M:0;71d8d2d6408d:40957 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T03:55:12,990 INFO [M:0;71d8d2d6408d:40957 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T03:55:12,990 DEBUG [M:0;71d8d2d6408d:40957 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T03:55:12,990 DEBUG [M:0;71d8d2d6408d:40957 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T03:55:12,991 DEBUG [M:0;71d8d2d6408d:40957 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T03:55:12,991 INFO [M:0;71d8d2d6408d:40957 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.67 KB heapSize=43.02 KB 2024-11-24T03:55:12,999 DEBUG [M:0;71d8d2d6408d:40957 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 44deece341184761bbc3ec165c722e69 2024-11-24T03:55:13,012 DEBUG [M:0;71d8d2d6408d:40957 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/44deece341184761bbc3ec165c722e69 is 82, key is hbase:meta,,1/info:regioninfo/1732420510019/Put/seqid=0 2024-11-24T03:55:13,012 INFO [M:0;71d8d2d6408d:40957 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/44deece341184761bbc3ec165c722e69 2024-11-24T03:55:13,020 DEBUG [M:0;71d8d2d6408d:40957 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on e533cdc0850746d6987a62f6c6365842 2024-11-24T03:55:13,033 DEBUG [M:0;71d8d2d6408d:40957 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e533cdc0850746d6987a62f6c6365842 is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732420510744/Put/seqid=0 2024-11-24T03:55:13,034 INFO [M:0;71d8d2d6408d:40957 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.05 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e533cdc0850746d6987a62f6c6365842 2024-11-24T03:55:13,042 DEBUG [M:0;71d8d2d6408d:40957 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8442ed554c5c4d23abf6a0f2320134a1 2024-11-24T03:55:13,059 DEBUG [M:0;71d8d2d6408d:40957 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8442ed554c5c4d23abf6a0f2320134a1 is 69, key is 71d8d2d6408d,36947,1732420508038/rs:state/1732420509260/Put/seqid=0 2024-11-24T03:55:13,060 INFO [M:0;71d8d2d6408d:40957 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8442ed554c5c4d23abf6a0f2320134a1 2024-11-24T03:55:13,062 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36947-0x1016c44a74a0002, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T03:55:13,062 INFO [RS:1;71d8d2d6408d:36947 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T03:55:13,062 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36947-0x1016c44a74a0002, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T03:55:13,062 INFO [RS:1;71d8d2d6408d:36947 {}] regionserver.HRegionServer(1031): Exiting; stopping=71d8d2d6408d,36947,1732420508038; zookeeper connection closed. 2024-11-24T03:55:13,063 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@97684a4 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@97684a4 2024-11-24T03:55:13,063 DEBUG [M:0;71d8d2d6408d:40957 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/44deece341184761bbc3ec165c722e69 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/44deece341184761bbc3ec165c722e69 2024-11-24T03:55:13,063 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-24T03:55:13,065 INFO [M:0;71d8d2d6408d:40957 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/44deece341184761bbc3ec165c722e69, entries=8, sequenceid=84, filesize=5.5 K 2024-11-24T03:55:13,066 DEBUG [M:0;71d8d2d6408d:40957 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e533cdc0850746d6987a62f6c6365842 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e533cdc0850746d6987a62f6c6365842 2024-11-24T03:55:13,068 INFO [M:0;71d8d2d6408d:40957 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e533cdc0850746d6987a62f6c6365842, entries=9, sequenceid=84, filesize=6.5 K 2024-11-24T03:55:13,069 DEBUG [M:0;71d8d2d6408d:40957 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8442ed554c5c4d23abf6a0f2320134a1 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8442ed554c5c4d23abf6a0f2320134a1 2024-11-24T03:55:13,071 INFO [M:0;71d8d2d6408d:40957 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8442ed554c5c4d23abf6a0f2320134a1, entries=2, sequenceid=84, filesize=5.1 K 2024-11-24T03:55:13,073 INFO [M:0;71d8d2d6408d:40957 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 81ms, sequenceid=84, compaction requested=false 2024-11-24T03:55:13,080 INFO [M:0;71d8d2d6408d:40957 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T03:55:13,081 DEBUG [M:0;71d8d2d6408d:40957 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732420512990Disabling compacts and flushes for region at 1732420512990Disabling writes for close at 1732420512991 (+1 ms)Obtaining lock to block concurrent updates at 1732420512991Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732420512991Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35506, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1732420512991Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732420512991Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732420512992 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732420513011 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732420513011Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732420513014 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732420513033 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732420513033Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732420513036 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732420513059 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732420513059Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d2ee87e: reopening flushed file at 1732420513062 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a80c17b: reopening flushed file at 1732420513066 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c24cb7d: reopening flushed file at 1732420513068 (+2 ms)Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 81ms, sequenceid=84, compaction requested=false at 1732420513073 (+5 ms)Writing region close event to WAL at 1732420513080 (+7 ms)Closed at 1732420513080 2024-11-24T03:55:13,083 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T03:55:13,083 INFO [M:0;71d8d2d6408d:40957 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-24T03:55:13,083 INFO [M:0;71d8d2d6408d:40957 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40957 2024-11-24T03:55:13,083 INFO [M:0;71d8d2d6408d:40957 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T03:55:13,213 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40957-0x1016c44a74a0000, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T03:55:13,213 INFO [M:0;71d8d2d6408d:40957 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T03:55:13,213 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40957-0x1016c44a74a0000, quorum=127.0.0.1:59166, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T03:55:13,222 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-24T03:55:13,244 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-24T03:55:13,244 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T03:55:13,251 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=51 (was 12) Potentially hanging thread: master/71d8d2d6408d:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: regionserver/71d8d2d6408d:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732420512813 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732420512635 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/71d8d2d6408d:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732420511082 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/71d8d2d6408d:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732420512816 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=244 (was 256), ProcessCount=11 (was 11), AvailableMemoryMB=7841 (was 8248) 2024-11-24T03:55:13,256 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=51, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=244, ProcessCount=11, AvailableMemoryMB=7842 2024-11-24T03:55:13,257 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/hadoop-log-dir so I do NOT create it in target/test-data/9882d271-12f2-7101-b4a3-bbc86c31780c 2024-11-24T03:55:13,257 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/hadoop-tmp-dir so I do NOT create it in target/test-data/9882d271-12f2-7101-b4a3-bbc86c31780c 2024-11-24T03:55:13,257 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T03:55:13,259 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9882d271-12f2-7101-b4a3-bbc86c31780c/cluster_7cbb415d-e42b-799b-e057-f2703bd80fac, deleteOnExit=true 2024-11-24T03:55:13,261 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9882d271-12f2-7101-b4a3-bbc86c31780c/cluster_7cbb415d-e42b-799b-e057-f2703bd80fac/zookeeper_0, clientPort=49981, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9882d271-12f2-7101-b4a3-bbc86c31780c/cluster_7cbb415d-e42b-799b-e057-f2703bd80fac/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9882d271-12f2-7101-b4a3-bbc86c31780c/cluster_7cbb415d-e42b-799b-e057-f2703bd80fac/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-24T03:55:13,262 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49981 2024-11-24T03:55:13,274 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b with version=8 2024-11-24T03:55:13,275 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/hbase-staging 2024-11-24T03:55:13,277 INFO [Time-limited test {}] client.ConnectionUtils(128): master/71d8d2d6408d:0 server-side Connection retries=45 2024-11-24T03:55:13,277 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T03:55:13,277 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T03:55:13,277 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T03:55:13,278 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T03:55:13,278 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T03:55:13,278 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-24T03:55:13,278 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T03:55:13,278 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33121 2024-11-24T03:55:13,280 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33121 connecting to ZooKeeper ensemble=127.0.0.1:49981 2024-11-24T03:55:13,316 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:331210x0, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T03:55:13,317 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33121-0x1016c44c0b20000 connected 2024-11-24T03:55:13,396 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T03:55:13,396 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T03:55:13,398 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33121-0x1016c44c0b20000, quorum=127.0.0.1:49981, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T03:55:13,398 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b, hbase.cluster.distributed=false 2024-11-24T03:55:13,400 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33121-0x1016c44c0b20000, quorum=127.0.0.1:49981, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T03:55:13,401 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33121 2024-11-24T03:55:13,401 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33121 2024-11-24T03:55:13,401 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33121 2024-11-24T03:55:13,402 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33121 2024-11-24T03:55:13,402 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33121 2024-11-24T03:55:13,422 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/71d8d2d6408d:0 server-side Connection retries=45 2024-11-24T03:55:13,422 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T03:55:13,422 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T03:55:13,422 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T03:55:13,422 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T03:55:13,422 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T03:55:13,422 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T03:55:13,422 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T03:55:13,423 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33751 2024-11-24T03:55:13,424 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33751 connecting to ZooKeeper ensemble=127.0.0.1:49981 2024-11-24T03:55:13,425 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T03:55:13,425 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T03:55:13,437 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:337510x0, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T03:55:13,437 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33751-0x1016c44c0b20001, quorum=127.0.0.1:49981, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T03:55:13,437 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33751-0x1016c44c0b20001 connected 2024-11-24T03:55:13,438 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T03:55:13,438 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9882d271-12f2-7101-b4a3-bbc86c31780c/bucket.cache 2024-11-24T03:55:13,438 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-24T03:55:13,439 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-24T03:55:13,439 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-24T03:55:13,439 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-24T03:55:13,439 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-24T03:55:13,439 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9882d271-12f2-7101-b4a3-bbc86c31780c/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9882d271-12f2-7101-b4a3-bbc86c31780c/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-24T03:55:13,439 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T03:55:13,440 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-24T03:55:13,440 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9882d271-12f2-7101-b4a3-bbc86c31780c/bucket.persistence is complete. 2024-11-24T03:55:13,440 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33751-0x1016c44c0b20001, quorum=127.0.0.1:49981, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T03:55:13,442 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33751-0x1016c44c0b20001, quorum=127.0.0.1:49981, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T03:55:13,442 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33751 2024-11-24T03:55:13,442 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33751 2024-11-24T03:55:13,443 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33751 2024-11-24T03:55:13,444 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33751 2024-11-24T03:55:13,444 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33751 2024-11-24T03:55:13,462 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/71d8d2d6408d:0 server-side Connection retries=45 2024-11-24T03:55:13,462 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T03:55:13,462 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T03:55:13,462 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T03:55:13,462 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T03:55:13,462 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T03:55:13,462 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T03:55:13,462 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T03:55:13,463 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45285 2024-11-24T03:55:13,465 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45285 connecting to ZooKeeper ensemble=127.0.0.1:49981 2024-11-24T03:55:13,465 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T03:55:13,465 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T03:55:13,471 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:452850x0, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T03:55:13,471 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:452850x0, quorum=127.0.0.1:49981, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T03:55:13,472 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45285-0x1016c44c0b20002 connected 2024-11-24T03:55:13,472 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T03:55:13,472 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9882d271-12f2-7101-b4a3-bbc86c31780c/bucket.cache 2024-11-24T03:55:13,472 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-24T03:55:13,473 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-24T03:55:13,473 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-24T03:55:13,474 INFO [Thread-121 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-24T03:55:13,475 WARN [Thread-121 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-24T03:55:13,475 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9882d271-12f2-7101-b4a3-bbc86c31780c/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9882d271-12f2-7101-b4a3-bbc86c31780c/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-24T03:55:13,475 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T03:55:13,475 INFO [Thread-121 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-24T03:55:13,475 INFO [Thread-121 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9882d271-12f2-7101-b4a3-bbc86c31780c/bucket.persistence is complete. 2024-11-24T03:55:13,476 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45285-0x1016c44c0b20002, quorum=127.0.0.1:49981, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T03:55:13,477 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45285-0x1016c44c0b20002, quorum=127.0.0.1:49981, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T03:55:13,478 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45285 2024-11-24T03:55:13,478 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45285 2024-11-24T03:55:13,478 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45285 2024-11-24T03:55:13,480 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45285 2024-11-24T03:55:13,480 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45285 2024-11-24T03:55:13,491 DEBUG [M:0;71d8d2d6408d:33121 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;71d8d2d6408d:33121 2024-11-24T03:55:13,492 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/71d8d2d6408d,33121,1732420513277 2024-11-24T03:55:13,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33121-0x1016c44c0b20000, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T03:55:13,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45285-0x1016c44c0b20002, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T03:55:13,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33751-0x1016c44c0b20001, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T03:55:13,497 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33121-0x1016c44c0b20000, quorum=127.0.0.1:49981, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/71d8d2d6408d,33121,1732420513277 2024-11-24T03:55:13,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33751-0x1016c44c0b20001, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T03:55:13,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45285-0x1016c44c0b20002, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T03:55:13,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33121-0x1016c44c0b20000, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T03:55:13,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45285-0x1016c44c0b20002, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T03:55:13,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33751-0x1016c44c0b20001, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T03:55:13,505 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33121-0x1016c44c0b20000, quorum=127.0.0.1:49981, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-24T03:55:13,505 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/71d8d2d6408d,33121,1732420513277 from backup master directory 2024-11-24T03:55:13,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33121-0x1016c44c0b20000, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/71d8d2d6408d,33121,1732420513277 2024-11-24T03:55:13,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33751-0x1016c44c0b20001, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T03:55:13,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45285-0x1016c44c0b20002, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T03:55:13,513 WARN [master/71d8d2d6408d:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T03:55:13,513 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33121-0x1016c44c0b20000, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T03:55:13,513 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=71d8d2d6408d,33121,1732420513277 2024-11-24T03:55:13,514 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/hbase.id] with ID: 11701ec0-a090-4213-af12-cdbaeca38d6a 2024-11-24T03:55:13,514 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/.tmp/hbase.id 2024-11-24T03:55:13,517 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/hbase.id] 2024-11-24T03:55:13,545 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T03:55:13,545 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-24T03:55:13,548 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 3ms. 2024-11-24T03:55:13,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33751-0x1016c44c0b20001, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T03:55:13,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45285-0x1016c44c0b20002, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T03:55:13,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33121-0x1016c44c0b20000, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T03:55:13,559 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T03:55:13,560 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-24T03:55:13,561 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T03:55:13,579 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/MasterData/data/master/store 2024-11-24T03:55:13,586 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T03:55:13,587 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T03:55:13,587 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T03:55:13,587 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T03:55:13,587 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T03:55:13,587 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T03:55:13,587 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T03:55:13,587 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732420513587Disabling compacts and flushes for region at 1732420513587Disabling writes for close at 1732420513587Writing region close event to WAL at 1732420513587Closed at 1732420513587 2024-11-24T03:55:13,588 WARN [master/71d8d2d6408d:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/MasterData/data/master/store/.initializing 2024-11-24T03:55:13,588 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/MasterData/WALs/71d8d2d6408d,33121,1732420513277 2024-11-24T03:55:13,594 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T03:55:13,598 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=71d8d2d6408d%2C33121%2C1732420513277, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/MasterData/WALs/71d8d2d6408d,33121,1732420513277, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/MasterData/oldWALs, maxLogs=10 2024-11-24T03:55:13,614 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/MasterData/WALs/71d8d2d6408d,33121,1732420513277/71d8d2d6408d%2C33121%2C1732420513277.1732420513598 2024-11-24T03:55:13,615 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T03:55:13,615 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-24T03:55:13,615 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T03:55:13,615 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T03:55:13,615 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T03:55:13,616 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T03:55:13,620 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-24T03:55:13,620 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T03:55:13,620 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T03:55:13,620 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T03:55:13,623 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-24T03:55:13,623 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T03:55:13,624 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T03:55:13,624 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T03:55:13,628 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-24T03:55:13,628 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T03:55:13,628 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T03:55:13,628 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T03:55:13,631 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-24T03:55:13,631 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T03:55:13,632 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T03:55:13,632 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T03:55:13,632 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T03:55:13,632 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T03:55:13,633 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T03:55:13,633 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T03:55:13,633 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T03:55:13,633 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T03:55:13,638 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T03:55:13,638 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73982610, jitterRate=0.10242679715156555}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T03:55:13,639 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732420513616Initializing all the Stores at 1732420513616Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732420513616Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732420513616Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732420513616Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732420513616Cleaning up temporary data from old regions at 1732420513633 (+17 ms)Region opened successfully at 1732420513639 (+6 ms) 2024-11-24T03:55:13,640 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-24T03:55:13,643 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c3dec38, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=71d8d2d6408d/172.17.0.2:0 2024-11-24T03:55:13,644 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-24T03:55:13,644 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-24T03:55:13,644 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-24T03:55:13,645 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T03:55:13,645 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T03:55:13,646 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-24T03:55:13,646 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-24T03:55:13,648 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-24T03:55:13,649 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33121-0x1016c44c0b20000, quorum=127.0.0.1:49981, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-24T03:55:13,678 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-24T03:55:13,679 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-24T03:55:13,680 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33121-0x1016c44c0b20000, quorum=127.0.0.1:49981, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-24T03:55:13,687 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-24T03:55:13,688 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-24T03:55:13,689 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33121-0x1016c44c0b20000, quorum=127.0.0.1:49981, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-24T03:55:13,696 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-24T03:55:13,697 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33121-0x1016c44c0b20000, quorum=127.0.0.1:49981, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-24T03:55:13,704 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-24T03:55:13,707 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33121-0x1016c44c0b20000, quorum=127.0.0.1:49981, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-24T03:55:13,712 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-24T03:55:13,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33121-0x1016c44c0b20000, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T03:55:13,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33751-0x1016c44c0b20001, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T03:55:13,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45285-0x1016c44c0b20002, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T03:55:13,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33121-0x1016c44c0b20000, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T03:55:13,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33751-0x1016c44c0b20001, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T03:55:13,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45285-0x1016c44c0b20002, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T03:55:13,722 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=71d8d2d6408d,33121,1732420513277, sessionid=0x1016c44c0b20000, setting cluster-up flag (Was=false) 2024-11-24T03:55:13,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45285-0x1016c44c0b20002, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T03:55:13,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33121-0x1016c44c0b20000, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T03:55:13,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33751-0x1016c44c0b20001, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T03:55:13,763 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-24T03:55:13,764 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=71d8d2d6408d,33121,1732420513277 2024-11-24T03:55:13,779 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33751-0x1016c44c0b20001, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T03:55:13,779 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33121-0x1016c44c0b20000, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T03:55:13,779 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45285-0x1016c44c0b20002, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T03:55:13,804 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-24T03:55:13,806 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=71d8d2d6408d,33121,1732420513277 2024-11-24T03:55:13,806 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-24T03:55:13,808 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T03:55:13,808 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-24T03:55:13,809 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-24T03:55:13,809 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 71d8d2d6408d,33121,1732420513277 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-24T03:55:13,811 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/71d8d2d6408d:0, corePoolSize=5, maxPoolSize=5 2024-11-24T03:55:13,811 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/71d8d2d6408d:0, corePoolSize=5, maxPoolSize=5 2024-11-24T03:55:13,811 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/71d8d2d6408d:0, corePoolSize=5, maxPoolSize=5 2024-11-24T03:55:13,812 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/71d8d2d6408d:0, corePoolSize=5, maxPoolSize=5 2024-11-24T03:55:13,812 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/71d8d2d6408d:0, corePoolSize=10, maxPoolSize=10 2024-11-24T03:55:13,812 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:13,812 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/71d8d2d6408d:0, corePoolSize=2, maxPoolSize=2 2024-11-24T03:55:13,812 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:13,813 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732420543813 2024-11-24T03:55:13,813 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-24T03:55:13,814 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-24T03:55:13,814 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-24T03:55:13,814 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-24T03:55:13,814 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-24T03:55:13,814 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-24T03:55:13,814 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T03:55:13,814 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-24T03:55:13,814 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:13,814 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-24T03:55:13,815 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T03:55:13,815 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-24T03:55:13,815 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-24T03:55:13,815 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-24T03:55:13,815 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-24T03:55:13,815 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-24T03:55:13,816 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/71d8d2d6408d:0:becomeActiveMaster-HFileCleaner.large.0-1732420513815,5,FailOnTimeoutGroup] 2024-11-24T03:55:13,816 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/71d8d2d6408d:0:becomeActiveMaster-HFileCleaner.small.0-1732420513816,5,FailOnTimeoutGroup] 2024-11-24T03:55:13,816 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:13,816 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-24T03:55:13,817 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:13,817 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:13,826 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-24T03:55:13,826 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b 2024-11-24T03:55:13,831 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T03:55:13,831 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-24T03:55:13,834 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T03:55:13,835 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T03:55:13,835 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T03:55:13,835 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-24T03:55:13,837 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T03:55:13,838 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T03:55:13,838 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T03:55:13,838 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-24T03:55:13,840 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T03:55:13,840 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T03:55:13,840 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T03:55:13,841 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-24T03:55:13,844 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T03:55:13,844 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T03:55:13,844 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T03:55:13,844 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T03:55:13,845 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740 2024-11-24T03:55:13,845 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740 2024-11-24T03:55:13,845 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T03:55:13,845 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T03:55:13,845 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T03:55:13,846 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T03:55:13,851 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T03:55:13,852 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67820685, jitterRate=0.010606959462165833}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T03:55:13,852 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732420513831Initializing all the Stores at 1732420513831Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732420513831Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732420513831Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732420513831Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732420513831Cleaning up temporary data from old regions at 1732420513845 (+14 ms)Region opened successfully at 1732420513852 (+7 ms) 2024-11-24T03:55:13,852 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T03:55:13,852 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T03:55:13,853 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T03:55:13,853 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T03:55:13,853 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T03:55:13,853 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T03:55:13,853 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732420513852Disabling compacts and flushes for region at 1732420513852Disabling writes for close at 1732420513853 (+1 ms)Writing region close event to WAL at 1732420513853Closed at 1732420513853 2024-11-24T03:55:13,854 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T03:55:13,855 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-24T03:55:13,855 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-24T03:55:13,856 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T03:55:13,857 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-24T03:55:13,883 INFO [RS:1;71d8d2d6408d:45285 {}] regionserver.HRegionServer(746): ClusterId : 11701ec0-a090-4213-af12-cdbaeca38d6a 2024-11-24T03:55:13,883 INFO [RS:0;71d8d2d6408d:33751 {}] regionserver.HRegionServer(746): ClusterId : 11701ec0-a090-4213-af12-cdbaeca38d6a 2024-11-24T03:55:13,883 DEBUG [RS:0;71d8d2d6408d:33751 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T03:55:13,883 DEBUG [RS:1;71d8d2d6408d:45285 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T03:55:13,906 DEBUG [RS:0;71d8d2d6408d:33751 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T03:55:13,906 DEBUG [RS:1;71d8d2d6408d:45285 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T03:55:13,906 DEBUG [RS:0;71d8d2d6408d:33751 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T03:55:13,906 DEBUG [RS:1;71d8d2d6408d:45285 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T03:55:13,914 DEBUG [RS:0;71d8d2d6408d:33751 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T03:55:13,914 DEBUG [RS:1;71d8d2d6408d:45285 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T03:55:13,914 DEBUG [RS:0;71d8d2d6408d:33751 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f906167, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=71d8d2d6408d/172.17.0.2:0 2024-11-24T03:55:13,914 DEBUG [RS:1;71d8d2d6408d:45285 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3aba8e80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=71d8d2d6408d/172.17.0.2:0 2024-11-24T03:55:13,927 DEBUG [RS:1;71d8d2d6408d:45285 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;71d8d2d6408d:45285 2024-11-24T03:55:13,927 INFO [RS:1;71d8d2d6408d:45285 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T03:55:13,927 INFO [RS:1;71d8d2d6408d:45285 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T03:55:13,927 DEBUG [RS:1;71d8d2d6408d:45285 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T03:55:13,927 DEBUG [RS:0;71d8d2d6408d:33751 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;71d8d2d6408d:33751 2024-11-24T03:55:13,927 INFO [RS:0;71d8d2d6408d:33751 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T03:55:13,927 INFO [RS:0;71d8d2d6408d:33751 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T03:55:13,927 DEBUG [RS:0;71d8d2d6408d:33751 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T03:55:13,928 INFO [RS:1;71d8d2d6408d:45285 {}] regionserver.HRegionServer(2659): reportForDuty to master=71d8d2d6408d,33121,1732420513277 with port=45285, startcode=1732420513461 2024-11-24T03:55:13,928 INFO [RS:0;71d8d2d6408d:33751 {}] regionserver.HRegionServer(2659): reportForDuty to master=71d8d2d6408d,33121,1732420513277 with port=33751, startcode=1732420513421 2024-11-24T03:55:13,928 DEBUG [RS:1;71d8d2d6408d:45285 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T03:55:13,928 DEBUG [RS:0;71d8d2d6408d:33751 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T03:55:13,931 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55113, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T03:55:13,931 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56965, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T03:55:13,931 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33121 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 71d8d2d6408d,33751,1732420513421 2024-11-24T03:55:13,931 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33121 {}] master.ServerManager(517): Registering regionserver=71d8d2d6408d,33751,1732420513421 2024-11-24T03:55:13,933 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33121 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 71d8d2d6408d,45285,1732420513461 2024-11-24T03:55:13,933 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33121 {}] master.ServerManager(517): Registering regionserver=71d8d2d6408d,45285,1732420513461 2024-11-24T03:55:13,933 DEBUG [RS:0;71d8d2d6408d:33751 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b 2024-11-24T03:55:13,933 DEBUG [RS:0;71d8d2d6408d:33751 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-24T03:55:13,933 DEBUG [RS:0;71d8d2d6408d:33751 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T03:55:13,934 DEBUG [RS:1;71d8d2d6408d:45285 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b 2024-11-24T03:55:13,934 DEBUG [RS:1;71d8d2d6408d:45285 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-24T03:55:13,934 DEBUG [RS:1;71d8d2d6408d:45285 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T03:55:13,937 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33121-0x1016c44c0b20000, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T03:55:13,966 DEBUG [RS:0;71d8d2d6408d:33751 {}] zookeeper.ZKUtil(111): regionserver:33751-0x1016c44c0b20001, quorum=127.0.0.1:49981, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/71d8d2d6408d,33751,1732420513421 2024-11-24T03:55:13,967 WARN [RS:0;71d8d2d6408d:33751 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T03:55:13,967 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [71d8d2d6408d,33751,1732420513421] 2024-11-24T03:55:13,967 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [71d8d2d6408d,45285,1732420513461] 2024-11-24T03:55:13,967 INFO [RS:0;71d8d2d6408d:33751 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T03:55:13,967 DEBUG [RS:0;71d8d2d6408d:33751 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/WALs/71d8d2d6408d,33751,1732420513421 2024-11-24T03:55:13,967 DEBUG [RS:1;71d8d2d6408d:45285 {}] zookeeper.ZKUtil(111): regionserver:45285-0x1016c44c0b20002, quorum=127.0.0.1:49981, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/71d8d2d6408d,45285,1732420513461 2024-11-24T03:55:13,967 WARN [RS:1;71d8d2d6408d:45285 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T03:55:13,967 INFO [RS:1;71d8d2d6408d:45285 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T03:55:13,968 DEBUG [RS:1;71d8d2d6408d:45285 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/WALs/71d8d2d6408d,45285,1732420513461 2024-11-24T03:55:13,974 INFO [RS:0;71d8d2d6408d:33751 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T03:55:13,979 INFO [RS:1;71d8d2d6408d:45285 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T03:55:13,979 INFO [RS:0;71d8d2d6408d:33751 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T03:55:13,979 INFO [RS:0;71d8d2d6408d:33751 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T03:55:13,979 INFO [RS:0;71d8d2d6408d:33751 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:13,980 INFO [RS:0;71d8d2d6408d:33751 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T03:55:13,980 INFO [RS:0;71d8d2d6408d:33751 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T03:55:13,980 INFO [RS:1;71d8d2d6408d:45285 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T03:55:13,980 INFO [RS:0;71d8d2d6408d:33751 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:13,981 DEBUG [RS:0;71d8d2d6408d:33751 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:13,981 DEBUG [RS:0;71d8d2d6408d:33751 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:13,981 INFO [RS:1;71d8d2d6408d:45285 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T03:55:13,981 INFO [RS:1;71d8d2d6408d:45285 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:13,981 DEBUG [RS:0;71d8d2d6408d:33751 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:13,981 DEBUG [RS:0;71d8d2d6408d:33751 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:13,981 DEBUG [RS:0;71d8d2d6408d:33751 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:13,981 INFO [RS:1;71d8d2d6408d:45285 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T03:55:13,981 DEBUG [RS:0;71d8d2d6408d:33751 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/71d8d2d6408d:0, corePoolSize=2, maxPoolSize=2 2024-11-24T03:55:13,981 DEBUG [RS:0;71d8d2d6408d:33751 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:13,981 DEBUG [RS:0;71d8d2d6408d:33751 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:13,982 DEBUG [RS:0;71d8d2d6408d:33751 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:13,982 INFO [RS:1;71d8d2d6408d:45285 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T03:55:13,982 DEBUG [RS:0;71d8d2d6408d:33751 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:13,982 INFO [RS:1;71d8d2d6408d:45285 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:13,982 DEBUG [RS:0;71d8d2d6408d:33751 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:13,982 DEBUG [RS:1;71d8d2d6408d:45285 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:13,982 DEBUG [RS:0;71d8d2d6408d:33751 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:13,982 DEBUG [RS:1;71d8d2d6408d:45285 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:13,982 DEBUG [RS:0;71d8d2d6408d:33751 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/71d8d2d6408d:0, corePoolSize=3, maxPoolSize=3 2024-11-24T03:55:13,982 DEBUG [RS:1;71d8d2d6408d:45285 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:13,982 DEBUG [RS:1;71d8d2d6408d:45285 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:13,982 DEBUG [RS:0;71d8d2d6408d:33751 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/71d8d2d6408d:0, corePoolSize=3, maxPoolSize=3 2024-11-24T03:55:13,982 DEBUG [RS:1;71d8d2d6408d:45285 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:13,982 DEBUG [RS:1;71d8d2d6408d:45285 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/71d8d2d6408d:0, corePoolSize=2, maxPoolSize=2 2024-11-24T03:55:13,982 DEBUG [RS:1;71d8d2d6408d:45285 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:13,983 DEBUG [RS:1;71d8d2d6408d:45285 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:13,983 INFO [RS:0;71d8d2d6408d:33751 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:13,983 INFO [RS:0;71d8d2d6408d:33751 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:13,983 DEBUG [RS:1;71d8d2d6408d:45285 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:13,983 INFO [RS:0;71d8d2d6408d:33751 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:13,983 DEBUG [RS:1;71d8d2d6408d:45285 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:13,983 INFO [RS:0;71d8d2d6408d:33751 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:13,983 INFO [RS:0;71d8d2d6408d:33751 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:13,983 DEBUG [RS:1;71d8d2d6408d:45285 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:13,983 INFO [RS:0;71d8d2d6408d:33751 {}] hbase.ChoreService(168): Chore ScheduledChore name=71d8d2d6408d,33751,1732420513421-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T03:55:13,983 DEBUG [RS:1;71d8d2d6408d:45285 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:13,983 DEBUG [RS:1;71d8d2d6408d:45285 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/71d8d2d6408d:0, corePoolSize=3, maxPoolSize=3 2024-11-24T03:55:13,983 DEBUG [RS:1;71d8d2d6408d:45285 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/71d8d2d6408d:0, corePoolSize=3, maxPoolSize=3 2024-11-24T03:55:13,986 INFO [RS:1;71d8d2d6408d:45285 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:13,986 INFO [RS:1;71d8d2d6408d:45285 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:13,986 INFO [RS:1;71d8d2d6408d:45285 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:13,986 INFO [RS:1;71d8d2d6408d:45285 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:13,986 INFO [RS:1;71d8d2d6408d:45285 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:13,986 INFO [RS:1;71d8d2d6408d:45285 {}] hbase.ChoreService(168): Chore ScheduledChore name=71d8d2d6408d,45285,1732420513461-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T03:55:14,000 INFO [RS:0;71d8d2d6408d:33751 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T03:55:14,000 INFO [RS:0;71d8d2d6408d:33751 {}] hbase.ChoreService(168): Chore ScheduledChore name=71d8d2d6408d,33751,1732420513421-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:14,000 INFO [RS:0;71d8d2d6408d:33751 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:14,000 INFO [RS:0;71d8d2d6408d:33751 {}] regionserver.Replication(171): 71d8d2d6408d,33751,1732420513421 started 2024-11-24T03:55:14,001 INFO [RS:1;71d8d2d6408d:45285 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T03:55:14,002 INFO [RS:1;71d8d2d6408d:45285 {}] hbase.ChoreService(168): Chore ScheduledChore name=71d8d2d6408d,45285,1732420513461-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:14,002 INFO [RS:1;71d8d2d6408d:45285 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:14,002 INFO [RS:1;71d8d2d6408d:45285 {}] regionserver.Replication(171): 71d8d2d6408d,45285,1732420513461 started 2024-11-24T03:55:14,007 WARN [71d8d2d6408d:33121 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T03:55:14,014 INFO [RS:0;71d8d2d6408d:33751 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:14,015 INFO [RS:0;71d8d2d6408d:33751 {}] regionserver.HRegionServer(1482): Serving as 71d8d2d6408d,33751,1732420513421, RpcServer on 71d8d2d6408d/172.17.0.2:33751, sessionid=0x1016c44c0b20001 2024-11-24T03:55:14,015 DEBUG [RS:0;71d8d2d6408d:33751 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T03:55:14,015 DEBUG [RS:0;71d8d2d6408d:33751 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 71d8d2d6408d,33751,1732420513421 2024-11-24T03:55:14,015 DEBUG [RS:0;71d8d2d6408d:33751 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '71d8d2d6408d,33751,1732420513421' 2024-11-24T03:55:14,015 DEBUG [RS:0;71d8d2d6408d:33751 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T03:55:14,016 DEBUG [RS:0;71d8d2d6408d:33751 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T03:55:14,016 DEBUG [RS:0;71d8d2d6408d:33751 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T03:55:14,016 DEBUG [RS:0;71d8d2d6408d:33751 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T03:55:14,016 DEBUG [RS:0;71d8d2d6408d:33751 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 71d8d2d6408d,33751,1732420513421 2024-11-24T03:55:14,016 DEBUG [RS:0;71d8d2d6408d:33751 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '71d8d2d6408d,33751,1732420513421' 2024-11-24T03:55:14,016 DEBUG [RS:0;71d8d2d6408d:33751 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T03:55:14,016 INFO [RS:1;71d8d2d6408d:45285 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:14,016 INFO [RS:1;71d8d2d6408d:45285 {}] regionserver.HRegionServer(1482): Serving as 71d8d2d6408d,45285,1732420513461, RpcServer on 71d8d2d6408d/172.17.0.2:45285, sessionid=0x1016c44c0b20002 2024-11-24T03:55:14,017 DEBUG [RS:1;71d8d2d6408d:45285 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T03:55:14,017 DEBUG [RS:1;71d8d2d6408d:45285 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 71d8d2d6408d,45285,1732420513461 2024-11-24T03:55:14,017 DEBUG [RS:0;71d8d2d6408d:33751 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T03:55:14,017 DEBUG [RS:1;71d8d2d6408d:45285 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '71d8d2d6408d,45285,1732420513461' 2024-11-24T03:55:14,017 DEBUG [RS:1;71d8d2d6408d:45285 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T03:55:14,017 DEBUG [RS:0;71d8d2d6408d:33751 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T03:55:14,017 INFO [RS:0;71d8d2d6408d:33751 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T03:55:14,017 INFO [RS:0;71d8d2d6408d:33751 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T03:55:14,017 DEBUG [RS:1;71d8d2d6408d:45285 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T03:55:14,018 DEBUG [RS:1;71d8d2d6408d:45285 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T03:55:14,018 DEBUG [RS:1;71d8d2d6408d:45285 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T03:55:14,018 DEBUG [RS:1;71d8d2d6408d:45285 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 71d8d2d6408d,45285,1732420513461 2024-11-24T03:55:14,018 DEBUG [RS:1;71d8d2d6408d:45285 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '71d8d2d6408d,45285,1732420513461' 2024-11-24T03:55:14,018 DEBUG [RS:1;71d8d2d6408d:45285 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T03:55:14,018 DEBUG [RS:1;71d8d2d6408d:45285 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T03:55:14,019 DEBUG [RS:1;71d8d2d6408d:45285 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T03:55:14,019 INFO [RS:1;71d8d2d6408d:45285 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T03:55:14,019 INFO [RS:1;71d8d2d6408d:45285 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T03:55:14,118 INFO [RS:0;71d8d2d6408d:33751 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T03:55:14,119 INFO [RS:0;71d8d2d6408d:33751 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=71d8d2d6408d%2C33751%2C1732420513421, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/WALs/71d8d2d6408d,33751,1732420513421, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/oldWALs, maxLogs=55 2024-11-24T03:55:14,119 INFO [RS:1;71d8d2d6408d:45285 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T03:55:14,120 INFO [RS:1;71d8d2d6408d:45285 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=71d8d2d6408d%2C45285%2C1732420513461, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/WALs/71d8d2d6408d,45285,1732420513461, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/oldWALs, maxLogs=55 2024-11-24T03:55:14,137 INFO [RS:0;71d8d2d6408d:33751 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/WALs/71d8d2d6408d,33751,1732420513421/71d8d2d6408d%2C33751%2C1732420513421.1732420514120 2024-11-24T03:55:14,138 INFO [RS:1;71d8d2d6408d:45285 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/WALs/71d8d2d6408d,45285,1732420513461/71d8d2d6408d%2C45285%2C1732420513461.1732420514121 2024-11-24T03:55:14,138 DEBUG [RS:0;71d8d2d6408d:33751 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T03:55:14,138 DEBUG [RS:1;71d8d2d6408d:45285 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T03:55:14,258 DEBUG [71d8d2d6408d:33121 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-24T03:55:14,258 DEBUG [71d8d2d6408d:33121 {}] balancer.BalancerClusterState(204): Hosts are {71d8d2d6408d=0} racks are {/default-rack=0} 2024-11-24T03:55:14,259 DEBUG [71d8d2d6408d:33121 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T03:55:14,260 DEBUG [71d8d2d6408d:33121 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T03:55:14,260 DEBUG [71d8d2d6408d:33121 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T03:55:14,260 DEBUG [71d8d2d6408d:33121 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T03:55:14,260 INFO [71d8d2d6408d:33121 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T03:55:14,260 INFO [71d8d2d6408d:33121 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T03:55:14,260 DEBUG [71d8d2d6408d:33121 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T03:55:14,261 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=71d8d2d6408d,33751,1732420513421 2024-11-24T03:55:14,263 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 71d8d2d6408d,33751,1732420513421, state=OPENING 2024-11-24T03:55:14,296 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-24T03:55:14,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33751-0x1016c44c0b20001, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T03:55:14,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45285-0x1016c44c0b20002, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T03:55:14,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33121-0x1016c44c0b20000, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T03:55:14,305 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T03:55:14,305 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T03:55:14,305 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T03:55:14,305 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T03:55:14,305 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=71d8d2d6408d,33751,1732420513421}] 2024-11-24T03:55:14,459 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T03:55:14,462 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44695, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T03:55:14,465 INFO [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-24T03:55:14,465 INFO [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T03:55:14,465 INFO [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-24T03:55:14,466 INFO [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=71d8d2d6408d%2C33751%2C1732420513421.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/WALs/71d8d2d6408d,33751,1732420513421, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/oldWALs, maxLogs=55 2024-11-24T03:55:14,485 INFO [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/WALs/71d8d2d6408d,33751,1732420513421/71d8d2d6408d%2C33751%2C1732420513421.meta.1732420514467.meta 2024-11-24T03:55:14,486 DEBUG [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T03:55:14,486 DEBUG [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-24T03:55:14,486 DEBUG [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-24T03:55:14,487 DEBUG [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-24T03:55:14,487 INFO [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-24T03:55:14,487 DEBUG [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-24T03:55:14,487 DEBUG [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T03:55:14,487 DEBUG [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-24T03:55:14,487 DEBUG [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-24T03:55:14,488 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-24T03:55:14,488 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T03:55:14,489 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T03:55:14,489 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T03:55:14,489 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-24T03:55:14,489 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T03:55:14,489 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T03:55:14,490 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T03:55:14,490 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-24T03:55:14,490 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T03:55:14,490 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T03:55:14,490 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T03:55:14,491 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-24T03:55:14,491 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T03:55:14,491 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T03:55:14,491 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T03:55:14,491 DEBUG [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T03:55:14,492 DEBUG [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740 2024-11-24T03:55:14,492 DEBUG [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740 2024-11-24T03:55:14,493 DEBUG [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T03:55:14,493 DEBUG [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T03:55:14,493 DEBUG [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T03:55:14,494 DEBUG [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T03:55:14,494 INFO [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75079915, jitterRate=0.11877791583538055}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T03:55:14,494 DEBUG [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-24T03:55:14,495 DEBUG [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732420514487Writing region info on filesystem at 1732420514487Initializing all the Stores at 1732420514487Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732420514488 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732420514488Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732420514488Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732420514488Cleaning up temporary data from old regions at 1732420514493 (+5 ms)Running coprocessor post-open hooks at 1732420514494 (+1 ms)Region opened successfully at 1732420514495 (+1 ms) 2024-11-24T03:55:14,495 INFO [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732420514459 2024-11-24T03:55:14,498 DEBUG [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-24T03:55:14,498 INFO [RS_OPEN_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-24T03:55:14,498 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=71d8d2d6408d,33751,1732420513421 2024-11-24T03:55:14,499 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 71d8d2d6408d,33751,1732420513421, state=OPEN 2024-11-24T03:55:14,528 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45285-0x1016c44c0b20002, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T03:55:14,528 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33751-0x1016c44c0b20001, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T03:55:14,528 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33121-0x1016c44c0b20000, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T03:55:14,528 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=71d8d2d6408d,33751,1732420513421 2024-11-24T03:55:14,529 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T03:55:14,529 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T03:55:14,529 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T03:55:14,531 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-24T03:55:14,531 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=71d8d2d6408d,33751,1732420513421 in 224 msec 2024-11-24T03:55:14,532 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-24T03:55:14,533 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 676 msec 2024-11-24T03:55:14,533 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T03:55:14,533 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-24T03:55:14,536 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T03:55:14,536 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=71d8d2d6408d,33751,1732420513421, seqNum=-1] 2024-11-24T03:55:14,536 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T03:55:14,538 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44839, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T03:55:14,543 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 735 msec 2024-11-24T03:55:14,543 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732420514543, completionTime=-1 2024-11-24T03:55:14,544 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-24T03:55:14,544 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T03:55:14,546 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T03:55:14,546 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732420574546 2024-11-24T03:55:14,546 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732420634546 2024-11-24T03:55:14,546 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-24T03:55:14,547 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=71d8d2d6408d,33121,1732420513277-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:14,547 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=71d8d2d6408d,33121,1732420513277-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:14,547 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=71d8d2d6408d,33121,1732420513277-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:14,547 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-71d8d2d6408d:33121, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:14,547 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:14,547 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:14,550 DEBUG [master/71d8d2d6408d:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-24T03:55:14,552 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.039sec 2024-11-24T03:55:14,553 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-24T03:55:14,553 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-24T03:55:14,553 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-24T03:55:14,553 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-24T03:55:14,553 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-24T03:55:14,553 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=71d8d2d6408d,33121,1732420513277-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T03:55:14,553 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=71d8d2d6408d,33121,1732420513277-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-24T03:55:14,554 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-24T03:55:14,554 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-24T03:55:14,554 INFO [master/71d8d2d6408d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=71d8d2d6408d,33121,1732420513277-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:14,583 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143f7fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T03:55:14,583 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 71d8d2d6408d,33121,-1 for getting cluster id 2024-11-24T03:55:14,583 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T03:55:14,585 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '11701ec0-a090-4213-af12-cdbaeca38d6a' 2024-11-24T03:55:14,585 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T03:55:14,586 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "11701ec0-a090-4213-af12-cdbaeca38d6a" 2024-11-24T03:55:14,586 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bc070b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T03:55:14,586 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [71d8d2d6408d,33121,-1] 2024-11-24T03:55:14,586 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T03:55:14,587 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T03:55:14,588 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39500, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T03:55:14,589 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2af0bbc0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T03:55:14,590 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T03:55:14,591 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=71d8d2d6408d,33751,1732420513421, seqNum=-1] 2024-11-24T03:55:14,591 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T03:55:14,594 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41270, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T03:55:14,596 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=71d8d2d6408d,33121,1732420513277 2024-11-24T03:55:14,596 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T03:55:14,598 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncConnectionImpl(321): The fetched master address is 71d8d2d6408d,33121,1732420513277 2024-11-24T03:55:14,598 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3552cc03 2024-11-24T03:55:14,598 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T03:55:14,600 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39502, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T03:55:14,601 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33121 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1732420514596', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T03:55:14,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33121 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732420514596 2024-11-24T03:55:14,603 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732420514596 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T03:55:14,603 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T03:55:14,603 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33121 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1732420514596" procId is: 4 2024-11-24T03:55:14,604 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732420514596 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T03:55:14,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33121 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T03:55:14,613 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732420514596-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 67da2bfde1dfe1c5a7387d8137ee444f, NAME => 'testBlockEvictionOnGracefulStop1732420514596,,1732420514600.67da2bfde1dfe1c5a7387d8137ee444f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1732420514596', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b 2024-11-24T03:55:14,620 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732420514596-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732420514596,,1732420514600.67da2bfde1dfe1c5a7387d8137ee444f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T03:55:14,620 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732420514596-pool-0 {}] regionserver.HRegion(1722): Closing 67da2bfde1dfe1c5a7387d8137ee444f, disabling compactions & flushes 2024-11-24T03:55:14,620 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732420514596-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732420514596,,1732420514600.67da2bfde1dfe1c5a7387d8137ee444f. 2024-11-24T03:55:14,620 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732420514596-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732420514596,,1732420514600.67da2bfde1dfe1c5a7387d8137ee444f. 2024-11-24T03:55:14,620 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732420514596-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732420514596,,1732420514600.67da2bfde1dfe1c5a7387d8137ee444f. after waiting 0 ms 2024-11-24T03:55:14,620 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732420514596-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732420514596,,1732420514600.67da2bfde1dfe1c5a7387d8137ee444f. 2024-11-24T03:55:14,620 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732420514596-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732420514596,,1732420514600.67da2bfde1dfe1c5a7387d8137ee444f. 2024-11-24T03:55:14,620 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732420514596-pool-0 {}] regionserver.HRegion(1676): Region close journal for 67da2bfde1dfe1c5a7387d8137ee444f: Waiting for close lock at 1732420514620Disabling compacts and flushes for region at 1732420514620Disabling writes for close at 1732420514620Writing region close event to WAL at 1732420514620Closed at 1732420514620 2024-11-24T03:55:14,622 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732420514596 execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T03:55:14,622 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1732420514596,,1732420514600.67da2bfde1dfe1c5a7387d8137ee444f.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732420514622"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732420514622"}]},"ts":"1732420514622"} 2024-11-24T03:55:14,625 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T03:55:14,626 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732420514596 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T03:55:14,626 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732420514596","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732420514626"}]},"ts":"1732420514626"} 2024-11-24T03:55:14,629 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732420514596, state=ENABLING in hbase:meta 2024-11-24T03:55:14,629 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {71d8d2d6408d=0} racks are {/default-rack=0} 2024-11-24T03:55:14,630 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T03:55:14,630 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T03:55:14,630 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T03:55:14,630 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T03:55:14,630 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T03:55:14,630 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T03:55:14,630 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T03:55:14,630 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732420514596, region=67da2bfde1dfe1c5a7387d8137ee444f, ASSIGN}] 2024-11-24T03:55:14,631 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732420514596, region=67da2bfde1dfe1c5a7387d8137ee444f, ASSIGN 2024-11-24T03:55:14,632 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732420514596, region=67da2bfde1dfe1c5a7387d8137ee444f, ASSIGN; state=OFFLINE, location=71d8d2d6408d,45285,1732420513461; forceNewPlan=false, retain=false 2024-11-24T03:55:14,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33121 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T03:55:14,783 INFO [71d8d2d6408d:33121 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T03:55:14,784 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=67da2bfde1dfe1c5a7387d8137ee444f, regionState=OPENING, regionLocation=71d8d2d6408d,45285,1732420513461 2024-11-24T03:55:14,789 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732420514596, region=67da2bfde1dfe1c5a7387d8137ee444f, ASSIGN because future has completed 2024-11-24T03:55:14,791 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 67da2bfde1dfe1c5a7387d8137ee444f, server=71d8d2d6408d,45285,1732420513461}] 2024-11-24T03:55:14,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33121 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T03:55:14,946 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T03:55:14,950 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57901, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T03:55:14,953 INFO [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732420514596,,1732420514600.67da2bfde1dfe1c5a7387d8137ee444f. 2024-11-24T03:55:14,953 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 67da2bfde1dfe1c5a7387d8137ee444f, NAME => 'testBlockEvictionOnGracefulStop1732420514596,,1732420514600.67da2bfde1dfe1c5a7387d8137ee444f.', STARTKEY => '', ENDKEY => ''} 2024-11-24T03:55:14,954 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732420514596 67da2bfde1dfe1c5a7387d8137ee444f 2024-11-24T03:55:14,954 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732420514596,,1732420514600.67da2bfde1dfe1c5a7387d8137ee444f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T03:55:14,954 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 67da2bfde1dfe1c5a7387d8137ee444f 2024-11-24T03:55:14,954 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 67da2bfde1dfe1c5a7387d8137ee444f 2024-11-24T03:55:14,956 INFO [StoreOpener-67da2bfde1dfe1c5a7387d8137ee444f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 67da2bfde1dfe1c5a7387d8137ee444f 2024-11-24T03:55:14,960 INFO [StoreOpener-67da2bfde1dfe1c5a7387d8137ee444f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 67da2bfde1dfe1c5a7387d8137ee444f columnFamilyName family 2024-11-24T03:55:14,960 DEBUG [StoreOpener-67da2bfde1dfe1c5a7387d8137ee444f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T03:55:14,960 INFO [StoreOpener-67da2bfde1dfe1c5a7387d8137ee444f-1 {}] regionserver.HStore(327): Store=67da2bfde1dfe1c5a7387d8137ee444f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T03:55:14,960 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 67da2bfde1dfe1c5a7387d8137ee444f 2024-11-24T03:55:14,961 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/default/testBlockEvictionOnGracefulStop1732420514596/67da2bfde1dfe1c5a7387d8137ee444f 2024-11-24T03:55:14,961 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/default/testBlockEvictionOnGracefulStop1732420514596/67da2bfde1dfe1c5a7387d8137ee444f 2024-11-24T03:55:14,961 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 67da2bfde1dfe1c5a7387d8137ee444f 2024-11-24T03:55:14,961 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 67da2bfde1dfe1c5a7387d8137ee444f 2024-11-24T03:55:14,962 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 67da2bfde1dfe1c5a7387d8137ee444f 2024-11-24T03:55:14,967 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/default/testBlockEvictionOnGracefulStop1732420514596/67da2bfde1dfe1c5a7387d8137ee444f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T03:55:14,967 INFO [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 67da2bfde1dfe1c5a7387d8137ee444f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61637409, jitterRate=-0.08153103291988373}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T03:55:14,967 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 67da2bfde1dfe1c5a7387d8137ee444f 2024-11-24T03:55:14,968 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 67da2bfde1dfe1c5a7387d8137ee444f: Running coprocessor pre-open hook at 1732420514955Writing region info on filesystem at 1732420514955Initializing all the Stores at 1732420514955Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732420514955Cleaning up temporary data from old regions at 1732420514961 (+6 ms)Running coprocessor post-open hooks at 1732420514967 (+6 ms)Region opened successfully at 1732420514968 (+1 ms) 2024-11-24T03:55:14,969 INFO [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732420514596,,1732420514600.67da2bfde1dfe1c5a7387d8137ee444f., pid=6, masterSystemTime=1732420514945 2024-11-24T03:55:14,972 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=67da2bfde1dfe1c5a7387d8137ee444f, regionState=OPEN, openSeqNum=2, regionLocation=71d8d2d6408d,45285,1732420513461 2024-11-24T03:55:14,972 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732420514596,,1732420514600.67da2bfde1dfe1c5a7387d8137ee444f. 2024-11-24T03:55:14,972 INFO [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732420514596,,1732420514600.67da2bfde1dfe1c5a7387d8137ee444f. 2024-11-24T03:55:14,975 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 67da2bfde1dfe1c5a7387d8137ee444f, server=71d8d2d6408d,45285,1732420513461 because future has completed 2024-11-24T03:55:14,979 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-24T03:55:14,979 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 67da2bfde1dfe1c5a7387d8137ee444f, server=71d8d2d6408d,45285,1732420513461 in 185 msec 2024-11-24T03:55:14,981 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-24T03:55:14,981 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732420514596, region=67da2bfde1dfe1c5a7387d8137ee444f, ASSIGN in 349 msec 2024-11-24T03:55:14,982 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732420514596 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T03:55:14,982 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732420514596","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732420514982"}]},"ts":"1732420514982"} 2024-11-24T03:55:14,984 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732420514596, state=ENABLED in hbase:meta 2024-11-24T03:55:14,985 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732420514596 execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T03:55:14,987 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732420514596 in 384 msec 2024-11-24T03:55:15,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33121 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T03:55:15,232 INFO [RPCClient-NioEventLoopGroup-5-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1732420514596 completed 2024-11-24T03:55:15,232 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1732420514596 get assigned. Timeout = 60000ms 2024-11-24T03:55:15,232 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T03:55:15,238 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1732420514596 assigned to meta. Checking AM states. 2024-11-24T03:55:15,238 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T03:55:15,238 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1732420514596 assigned. 2024-11-24T03:55:15,242 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1732420514596', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1732420514596,,1732420514600.67da2bfde1dfe1c5a7387d8137ee444f., hostname=71d8d2d6408d,45285,1732420513461, seqNum=2] 2024-11-24T03:55:15,243 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T03:55:15,245 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37178, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T03:55:15,248 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 67da2bfde1dfe1c5a7387d8137ee444f 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-24T03:55:15,256 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f6d06f91d32744028654ba7d19278a90 2024-11-24T03:55:15,268 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/default/testBlockEvictionOnGracefulStop1732420514596/67da2bfde1dfe1c5a7387d8137ee444f/.tmp/family/f6d06f91d32744028654ba7d19278a90 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-24T03:55:15,269 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/default/testBlockEvictionOnGracefulStop1732420514596/67da2bfde1dfe1c5a7387d8137ee444f/.tmp/family/f6d06f91d32744028654ba7d19278a90 2024-11-24T03:55:15,271 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/default/testBlockEvictionOnGracefulStop1732420514596/67da2bfde1dfe1c5a7387d8137ee444f/.tmp/family/f6d06f91d32744028654ba7d19278a90 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/default/testBlockEvictionOnGracefulStop1732420514596/67da2bfde1dfe1c5a7387d8137ee444f/family/f6d06f91d32744028654ba7d19278a90 2024-11-24T03:55:15,272 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/default/testBlockEvictionOnGracefulStop1732420514596/67da2bfde1dfe1c5a7387d8137ee444f/family/f6d06f91d32744028654ba7d19278a90, delay=998 ms 2024-11-24T03:55:15,273 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/default/testBlockEvictionOnGracefulStop1732420514596/67da2bfde1dfe1c5a7387d8137ee444f/family/f6d06f91d32744028654ba7d19278a90, entries=2, sequenceid=6, filesize=4.9 K 2024-11-24T03:55:15,274 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 67da2bfde1dfe1c5a7387d8137ee444f in 26ms, sequenceid=6, compaction requested=false 2024-11-24T03:55:15,274 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1732420514596' 2024-11-24T03:55:15,274 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 67da2bfde1dfe1c5a7387d8137ee444f: 2024-11-24T03:55:15,490 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T03:55:16,271 DEBUG [hfile-prefetch-1732420512635 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key f6d06f91d32744028654ba7d19278a90_0. 2024-11-24T03:55:16,272 DEBUG [hfile-prefetch-1732420512635 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key f6d06f91d32744028654ba7d19278a90_117. 2024-11-24T03:55:16,273 DEBUG [hfile-prefetch-1732420512635 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/default/testBlockEvictionOnGracefulStop1732420514596/67da2bfde1dfe1c5a7387d8137ee444f/family/f6d06f91d32744028654ba7d19278a90, with total blocks 2, and data blocks 1 2024-11-24T03:55:16,273 DEBUG [hfile-prefetch-1732420512635 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-24T03:55:16,273 DEBUG [hfile-prefetch-1732420512635 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/default/testBlockEvictionOnGracefulStop1732420514596/67da2bfde1dfe1c5a7387d8137ee444f/family/f6d06f91d32744028654ba7d19278a90, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-24T03:55:16,273 DEBUG [hfile-prefetch-1732420512635 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/default/testBlockEvictionOnGracefulStop1732420514596/67da2bfde1dfe1c5a7387d8137ee444f/family/f6d06f91d32744028654ba7d19278a90. Will sleep 100ms and try the verification again. 2024-11-24T03:55:16,275 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;71d8d2d6408d:45285,5,FailOnTimeoutGroup] 2024-11-24T03:55:16,275 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '71d8d2d6408d,45285,1732420513461' ***** 2024-11-24T03:55:16,275 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-11-24T03:55:16,276 INFO [RS:1;71d8d2d6408d:45285 {}] bucket.BucketCache(1738): Disabling cache 2024-11-24T03:55:16,288 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T03:55:16,289 INFO [regionserver/71d8d2d6408d:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T03:55:16,357 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-24T03:55:16,357 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-24T03:55:16,357 INFO [RS:1;71d8d2d6408d:45285 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9882d271-12f2-7101-b4a3-bbc86c31780c/bucket.persistence 2024-11-24T03:55:16,357 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-24T03:55:16,357 INFO [RS:1;71d8d2d6408d:45285 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-24T03:55:16,357 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-24T03:55:16,367 DEBUG [RS:1;71d8d2d6408d:45285 {}] bucket.BucketCache(1487): Thread RS:1;71d8d2d6408d:45285 started persisting bucket cache to file 2024-11-24T03:55:16,368 DEBUG [RS:1;71d8d2d6408d:45285 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T03:55:16,368 DEBUG [RS:1;71d8d2d6408d:45285 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T03:55:16,373 DEBUG [hfile-prefetch-1732420512635 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/default/testBlockEvictionOnGracefulStop1732420514596/67da2bfde1dfe1c5a7387d8137ee444f/family/f6d06f91d32744028654ba7d19278a90, with total blocks 2, and data blocks 1 2024-11-24T03:55:16,373 DEBUG [hfile-prefetch-1732420512635 {}] bucket.BucketCache(2285): iterating over 1 entries in the backing map 2024-11-24T03:55:16,373 DEBUG [hfile-prefetch-1732420512635 {}] bucket.BucketCache(2293): found block for file f6d06f91d32744028654ba7d19278a90 in the backing map. Acquiring read lock for offset 0 2024-11-24T03:55:16,374 DEBUG [hfile-prefetch-1732420512635 {}] bucket.BucketCache(2305): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/default/testBlockEvictionOnGracefulStop1732420514596/67da2bfde1dfe1c5a7387d8137ee444f/family/f6d06f91d32744028654ba7d19278a90 has now been fully cached. 2024-11-24T03:55:16,374 DEBUG [RS:1;71d8d2d6408d:45285 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-24T03:55:16,374 DEBUG [RS:1;71d8d2d6408d:45285 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1732420516368 2024-11-24T03:55:16,374 DEBUG [hfile-prefetch-1732420512635 {}] hfile.PrefetchExecutor(121): Prefetch completed for f6d06f91d32744028654ba7d19278a90 2024-11-24T03:55:16,374 DEBUG [RS:1;71d8d2d6408d:45285 {}] bucket.BucketCache(1509): Thread RS:1;71d8d2d6408d:45285 finished persisting bucket cache to file, renaming 2024-11-24T03:55:16,375 INFO [RS:1;71d8d2d6408d:45285 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T03:55:16,375 INFO [RS:1;71d8d2d6408d:45285 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T03:55:16,375 INFO [RS:1;71d8d2d6408d:45285 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T03:55:16,375 INFO [RS:1;71d8d2d6408d:45285 {}] regionserver.HRegionServer(3091): Received CLOSE for 67da2bfde1dfe1c5a7387d8137ee444f 2024-11-24T03:55:16,376 INFO [RS:1;71d8d2d6408d:45285 {}] regionserver.HRegionServer(959): stopping server 71d8d2d6408d,45285,1732420513461 2024-11-24T03:55:16,376 INFO [RS:1;71d8d2d6408d:45285 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T03:55:16,376 INFO [RS:1;71d8d2d6408d:45285 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;71d8d2d6408d:45285. 2024-11-24T03:55:16,376 INFO [RS_CLOSE_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732420514596,,1732420514600.67da2bfde1dfe1c5a7387d8137ee444f. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-24T03:55:16,376 DEBUG [RS:1;71d8d2d6408d:45285 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T03:55:16,376 DEBUG [RS_CLOSE_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 67da2bfde1dfe1c5a7387d8137ee444f, disabling compactions & flushes 2024-11-24T03:55:16,376 DEBUG [RS:1;71d8d2d6408d:45285 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T03:55:16,376 INFO [RS_CLOSE_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732420514596,,1732420514600.67da2bfde1dfe1c5a7387d8137ee444f. 2024-11-24T03:55:16,377 DEBUG [RS_CLOSE_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732420514596,,1732420514600.67da2bfde1dfe1c5a7387d8137ee444f. 2024-11-24T03:55:16,377 INFO [RS:1;71d8d2d6408d:45285 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-24T03:55:16,377 DEBUG [RS_CLOSE_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732420514596,,1732420514600.67da2bfde1dfe1c5a7387d8137ee444f. after waiting 0 ms 2024-11-24T03:55:16,377 DEBUG [RS:1;71d8d2d6408d:45285 {}] regionserver.HRegionServer(1325): Online Regions={67da2bfde1dfe1c5a7387d8137ee444f=testBlockEvictionOnGracefulStop1732420514596,,1732420514600.67da2bfde1dfe1c5a7387d8137ee444f.} 2024-11-24T03:55:16,377 DEBUG [RS_CLOSE_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732420514596,,1732420514600.67da2bfde1dfe1c5a7387d8137ee444f. 2024-11-24T03:55:16,377 DEBUG [RS:1;71d8d2d6408d:45285 {}] regionserver.HRegionServer(1351): Waiting on 67da2bfde1dfe1c5a7387d8137ee444f 2024-11-24T03:55:16,383 DEBUG [RS_CLOSE_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/default/testBlockEvictionOnGracefulStop1732420514596/67da2bfde1dfe1c5a7387d8137ee444f/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-24T03:55:16,383 INFO [RS_CLOSE_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732420514596,,1732420514600.67da2bfde1dfe1c5a7387d8137ee444f. 2024-11-24T03:55:16,383 DEBUG [RS_CLOSE_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 67da2bfde1dfe1c5a7387d8137ee444f: Waiting for close lock at 1732420516376Running coprocessor pre-close hooks at 1732420516376Disabling compacts and flushes for region at 1732420516376Disabling writes for close at 1732420516377 (+1 ms)Writing region close event to WAL at 1732420516378 (+1 ms)Running coprocessor post-close hooks at 1732420516383 (+5 ms)Closed at 1732420516383 2024-11-24T03:55:16,383 DEBUG [RS_CLOSE_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732420514596,,1732420514600.67da2bfde1dfe1c5a7387d8137ee444f. 2024-11-24T03:55:16,442 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-24T03:55:16,442 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-24T03:55:16,442 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T03:55:16,442 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T03:55:16,447 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-24T03:55:16,447 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732420516442 2024-11-24T03:55:16,447 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-24T03:55:16,577 INFO [RS:1;71d8d2d6408d:45285 {}] regionserver.HRegionServer(976): stopping server 71d8d2d6408d,45285,1732420513461; all regions closed. 2024-11-24T03:55:16,580 DEBUG [RS:1;71d8d2d6408d:45285 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/oldWALs 2024-11-24T03:55:16,580 INFO [RS:1;71d8d2d6408d:45285 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 71d8d2d6408d%2C45285%2C1732420513461:(num 1732420514121) 2024-11-24T03:55:16,580 DEBUG [RS:1;71d8d2d6408d:45285 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T03:55:16,580 INFO [RS:1;71d8d2d6408d:45285 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T03:55:16,580 INFO [RS:1;71d8d2d6408d:45285 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T03:55:16,580 INFO [RS:1;71d8d2d6408d:45285 {}] hbase.ChoreService(370): Chore service for: regionserver/71d8d2d6408d:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T03:55:16,580 INFO [RS:1;71d8d2d6408d:45285 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T03:55:16,580 INFO [RS:1;71d8d2d6408d:45285 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T03:55:16,580 INFO [regionserver/71d8d2d6408d:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T03:55:16,580 INFO [RS:1;71d8d2d6408d:45285 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T03:55:16,581 INFO [RS:1;71d8d2d6408d:45285 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T03:55:16,581 INFO [RS:1;71d8d2d6408d:45285 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45285 2024-11-24T03:55:16,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33121-0x1016c44c0b20000, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T03:55:16,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45285-0x1016c44c0b20002, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/71d8d2d6408d,45285,1732420513461 2024-11-24T03:55:16,595 INFO [RS:1;71d8d2d6408d:45285 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T03:55:16,604 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [71d8d2d6408d,45285,1732420513461] 2024-11-24T03:55:16,612 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/71d8d2d6408d,45285,1732420513461 already deleted, retry=false 2024-11-24T03:55:16,612 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 71d8d2d6408d,45285,1732420513461 on 71d8d2d6408d,33121,1732420513277 2024-11-24T03:55:16,616 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 71d8d2d6408d,45285,1732420513461, splitWal=true, meta=false 2024-11-24T03:55:16,618 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 71d8d2d6408d,45285,1732420513461 (carryingMeta=false) 71d8d2d6408d,45285,1732420513461/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@5eb13c05[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-24T03:55:16,619 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 71d8d2d6408d,45285,1732420513461, splitWal=true, meta=false 2024-11-24T03:55:16,620 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): 71d8d2d6408d,45285,1732420513461 had 1 regions 2024-11-24T03:55:16,621 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 71d8d2d6408d,45285,1732420513461, splitWal=true, meta=false, isMeta: false 2024-11-24T03:55:16,622 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/WALs/71d8d2d6408d,45285,1732420513461-splitting 2024-11-24T03:55:16,622 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/WALs/71d8d2d6408d,45285,1732420513461-splitting dir is empty, no logs to split. 2024-11-24T03:55:16,622 INFO [PEWorker-2 {}] master.SplitWALManager(105): 71d8d2d6408d,45285,1732420513461 WAL count=0, meta=false 2024-11-24T03:55:16,625 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/WALs/71d8d2d6408d,45285,1732420513461-splitting dir is empty, no logs to split. 2024-11-24T03:55:16,625 INFO [PEWorker-2 {}] master.SplitWALManager(105): 71d8d2d6408d,45285,1732420513461 WAL count=0, meta=false 2024-11-24T03:55:16,625 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 71d8d2d6408d,45285,1732420513461 WAL splitting is done? wals=0, meta=false 2024-11-24T03:55:16,625 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 71d8d2d6408d,45285,1732420513461 failed, ignore...File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/WALs/71d8d2d6408d,45285,1732420513461-splitting does not exist 2024-11-24T03:55:16,627 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732420514596, region=67da2bfde1dfe1c5a7387d8137ee444f, ASSIGN}] 2024-11-24T03:55:16,628 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732420514596, region=67da2bfde1dfe1c5a7387d8137ee444f, ASSIGN 2024-11-24T03:55:16,628 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732420514596, region=67da2bfde1dfe1c5a7387d8137ee444f, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-24T03:55:16,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45285-0x1016c44c0b20002, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T03:55:16,704 INFO [RS:1;71d8d2d6408d:45285 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T03:55:16,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45285-0x1016c44c0b20002, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T03:55:16,704 INFO [RS:1;71d8d2d6408d:45285 {}] regionserver.HRegionServer(1031): Exiting; stopping=71d8d2d6408d,45285,1732420513461; zookeeper connection closed. 2024-11-24T03:55:16,704 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6e20e8e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6e20e8e 2024-11-24T03:55:16,779 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=67da2bfde1dfe1c5a7387d8137ee444f, regionState=OPENING, regionLocation=71d8d2d6408d,33751,1732420513421 2024-11-24T03:55:16,784 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732420514596, region=67da2bfde1dfe1c5a7387d8137ee444f, ASSIGN because future has completed 2024-11-24T03:55:16,785 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 67da2bfde1dfe1c5a7387d8137ee444f, server=71d8d2d6408d,33751,1732420513421}] 2024-11-24T03:55:16,797 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/71d8d2d6408d:0 server-side Connection retries=45 2024-11-24T03:55:16,797 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T03:55:16,798 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T03:55:16,798 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T03:55:16,798 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T03:55:16,798 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T03:55:16,798 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T03:55:16,798 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T03:55:16,798 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39051 2024-11-24T03:55:16,800 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39051 connecting to ZooKeeper ensemble=127.0.0.1:49981 2024-11-24T03:55:16,800 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T03:55:16,800 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T03:55:16,820 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:390510x0, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T03:55:16,821 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:39051-0x1016c44c0b20003, quorum=127.0.0.1:49981, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-24T03:55:16,821 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39051-0x1016c44c0b20003 connected 2024-11-24T03:55:16,821 DEBUG [pool-297-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-24T03:55:16,822 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T03:55:16,823 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9882d271-12f2-7101-b4a3-bbc86c31780c/bucket.cache 2024-11-24T03:55:16,823 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-24T03:55:16,823 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-24T03:55:16,823 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-24T03:55:16,824 INFO [Thread-201 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-24T03:55:16,824 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9882d271-12f2-7101-b4a3-bbc86c31780c/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9882d271-12f2-7101-b4a3-bbc86c31780c/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-24T03:55:16,824 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T03:55:16,825 INFO [Thread-201 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-24T03:55:16,825 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:39051-0x1016c44c0b20003, quorum=127.0.0.1:49981, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-24T03:55:16,826 INFO [Thread-201 {}] bucket.BucketCache(1705): Retrieved 0 of chunks with blockCount = 0. 2024-11-24T03:55:16,828 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39051-0x1016c44c0b20003, quorum=127.0.0.1:49981, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T03:55:16,828 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39051 2024-11-24T03:55:16,829 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39051 2024-11-24T03:55:16,829 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39051 2024-11-24T03:55:16,829 DEBUG [Thread-201 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-24T03:55:16,830 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39051 2024-11-24T03:55:16,830 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39051 2024-11-24T03:55:16,830 INFO [Thread-201 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-24T03:55:16,830 INFO [Thread-201 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 0 2024-11-24T03:55:16,830 INFO [Thread-201 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9882d271-12f2-7101-b4a3-bbc86c31780c/bucket.persistence is complete. 2024-11-24T03:55:16,832 INFO [RS:2;71d8d2d6408d:39051 {}] regionserver.HRegionServer(746): ClusterId : 11701ec0-a090-4213-af12-cdbaeca38d6a 2024-11-24T03:55:16,832 DEBUG [RS:2;71d8d2d6408d:39051 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T03:55:16,838 DEBUG [RS:2;71d8d2d6408d:39051 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T03:55:16,838 DEBUG [RS:2;71d8d2d6408d:39051 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T03:55:16,846 DEBUG [RS:2;71d8d2d6408d:39051 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T03:55:16,846 DEBUG [RS:2;71d8d2d6408d:39051 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b4f4a76, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=71d8d2d6408d/172.17.0.2:0 2024-11-24T03:55:16,859 DEBUG [RS:2;71d8d2d6408d:39051 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;71d8d2d6408d:39051 2024-11-24T03:55:16,859 INFO [RS:2;71d8d2d6408d:39051 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T03:55:16,859 INFO [RS:2;71d8d2d6408d:39051 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T03:55:16,859 DEBUG [RS:2;71d8d2d6408d:39051 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T03:55:16,860 INFO [RS:2;71d8d2d6408d:39051 {}] regionserver.HRegionServer(2659): reportForDuty to master=71d8d2d6408d,33121,1732420513277 with port=39051, startcode=1732420516797 2024-11-24T03:55:16,860 DEBUG [RS:2;71d8d2d6408d:39051 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T03:55:16,862 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46497, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T03:55:16,862 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33121 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 71d8d2d6408d,39051,1732420516797 2024-11-24T03:55:16,862 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33121 {}] master.ServerManager(517): Registering regionserver=71d8d2d6408d,39051,1732420516797 2024-11-24T03:55:16,864 DEBUG [RS:2;71d8d2d6408d:39051 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b 2024-11-24T03:55:16,864 DEBUG [RS:2;71d8d2d6408d:39051 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-24T03:55:16,864 DEBUG [RS:2;71d8d2d6408d:39051 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T03:55:16,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33121-0x1016c44c0b20000, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T03:55:16,871 DEBUG [RS:2;71d8d2d6408d:39051 {}] zookeeper.ZKUtil(111): regionserver:39051-0x1016c44c0b20003, quorum=127.0.0.1:49981, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/71d8d2d6408d,39051,1732420516797 2024-11-24T03:55:16,871 WARN [RS:2;71d8d2d6408d:39051 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T03:55:16,871 INFO [RS:2;71d8d2d6408d:39051 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T03:55:16,871 DEBUG [RS:2;71d8d2d6408d:39051 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/WALs/71d8d2d6408d,39051,1732420516797 2024-11-24T03:55:16,871 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [71d8d2d6408d,39051,1732420516797] 2024-11-24T03:55:16,876 INFO [RS:2;71d8d2d6408d:39051 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T03:55:16,878 INFO [RS:2;71d8d2d6408d:39051 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T03:55:16,879 INFO [RS:2;71d8d2d6408d:39051 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T03:55:16,879 INFO [RS:2;71d8d2d6408d:39051 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:16,879 INFO [RS:2;71d8d2d6408d:39051 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T03:55:16,880 INFO [RS:2;71d8d2d6408d:39051 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T03:55:16,880 INFO [RS:2;71d8d2d6408d:39051 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:16,880 DEBUG [RS:2;71d8d2d6408d:39051 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:16,880 DEBUG [RS:2;71d8d2d6408d:39051 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:16,880 DEBUG [RS:2;71d8d2d6408d:39051 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:16,880 DEBUG [RS:2;71d8d2d6408d:39051 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:16,880 DEBUG [RS:2;71d8d2d6408d:39051 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:16,880 DEBUG [RS:2;71d8d2d6408d:39051 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/71d8d2d6408d:0, corePoolSize=2, maxPoolSize=2 2024-11-24T03:55:16,880 DEBUG [RS:2;71d8d2d6408d:39051 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:16,880 DEBUG [RS:2;71d8d2d6408d:39051 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:16,880 DEBUG [RS:2;71d8d2d6408d:39051 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:16,880 DEBUG [RS:2;71d8d2d6408d:39051 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:16,881 DEBUG [RS:2;71d8d2d6408d:39051 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:16,881 DEBUG [RS:2;71d8d2d6408d:39051 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/71d8d2d6408d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T03:55:16,881 DEBUG [RS:2;71d8d2d6408d:39051 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/71d8d2d6408d:0, corePoolSize=3, maxPoolSize=3 2024-11-24T03:55:16,881 DEBUG [RS:2;71d8d2d6408d:39051 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/71d8d2d6408d:0, corePoolSize=3, maxPoolSize=3 2024-11-24T03:55:16,881 INFO [RS:2;71d8d2d6408d:39051 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:16,881 INFO [RS:2;71d8d2d6408d:39051 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:16,881 INFO [RS:2;71d8d2d6408d:39051 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:16,881 INFO [RS:2;71d8d2d6408d:39051 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:16,881 INFO [RS:2;71d8d2d6408d:39051 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:16,881 INFO [RS:2;71d8d2d6408d:39051 {}] hbase.ChoreService(168): Chore ScheduledChore name=71d8d2d6408d,39051,1732420516797-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T03:55:16,897 INFO [RS:2;71d8d2d6408d:39051 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T03:55:16,898 INFO [RS:2;71d8d2d6408d:39051 {}] hbase.ChoreService(168): Chore ScheduledChore name=71d8d2d6408d,39051,1732420516797-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:16,898 INFO [RS:2;71d8d2d6408d:39051 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:16,898 INFO [RS:2;71d8d2d6408d:39051 {}] regionserver.Replication(171): 71d8d2d6408d,39051,1732420516797 started 2024-11-24T03:55:16,913 INFO [RS:2;71d8d2d6408d:39051 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T03:55:16,914 INFO [RS:2;71d8d2d6408d:39051 {}] regionserver.HRegionServer(1482): Serving as 71d8d2d6408d,39051,1732420516797, RpcServer on 71d8d2d6408d/172.17.0.2:39051, sessionid=0x1016c44c0b20003 2024-11-24T03:55:16,914 DEBUG [RS:2;71d8d2d6408d:39051 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T03:55:16,914 DEBUG [RS:2;71d8d2d6408d:39051 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 71d8d2d6408d,39051,1732420516797 2024-11-24T03:55:16,914 DEBUG [RS:2;71d8d2d6408d:39051 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '71d8d2d6408d,39051,1732420516797' 2024-11-24T03:55:16,914 DEBUG [RS:2;71d8d2d6408d:39051 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T03:55:16,915 DEBUG [RS:2;71d8d2d6408d:39051 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T03:55:16,915 DEBUG [RS:2;71d8d2d6408d:39051 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T03:55:16,916 DEBUG [RS:2;71d8d2d6408d:39051 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T03:55:16,916 DEBUG [RS:2;71d8d2d6408d:39051 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 71d8d2d6408d,39051,1732420516797 2024-11-24T03:55:16,916 DEBUG [RS:2;71d8d2d6408d:39051 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '71d8d2d6408d,39051,1732420516797' 2024-11-24T03:55:16,916 DEBUG [RS:2;71d8d2d6408d:39051 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T03:55:16,916 DEBUG [RS:2;71d8d2d6408d:39051 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T03:55:16,917 DEBUG [RS:2;71d8d2d6408d:39051 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T03:55:16,917 INFO [RS:2;71d8d2d6408d:39051 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T03:55:16,917 INFO [RS:2;71d8d2d6408d:39051 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T03:55:16,940 INFO [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732420514596,,1732420514600.67da2bfde1dfe1c5a7387d8137ee444f. 2024-11-24T03:55:16,941 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 67da2bfde1dfe1c5a7387d8137ee444f, NAME => 'testBlockEvictionOnGracefulStop1732420514596,,1732420514600.67da2bfde1dfe1c5a7387d8137ee444f.', STARTKEY => '', ENDKEY => ''} 2024-11-24T03:55:16,941 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732420514596 67da2bfde1dfe1c5a7387d8137ee444f 2024-11-24T03:55:16,941 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732420514596,,1732420514600.67da2bfde1dfe1c5a7387d8137ee444f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T03:55:16,941 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 67da2bfde1dfe1c5a7387d8137ee444f 2024-11-24T03:55:16,941 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 67da2bfde1dfe1c5a7387d8137ee444f 2024-11-24T03:55:16,942 INFO [StoreOpener-67da2bfde1dfe1c5a7387d8137ee444f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 67da2bfde1dfe1c5a7387d8137ee444f 2024-11-24T03:55:16,943 INFO [StoreOpener-67da2bfde1dfe1c5a7387d8137ee444f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 67da2bfde1dfe1c5a7387d8137ee444f columnFamilyName family 2024-11-24T03:55:16,943 DEBUG [StoreOpener-67da2bfde1dfe1c5a7387d8137ee444f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T03:55:16,945 DEBUG [StoreFileOpener-67da2bfde1dfe1c5a7387d8137ee444f-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/default/testBlockEvictionOnGracefulStop1732420514596/67da2bfde1dfe1c5a7387d8137ee444f/family/f6d06f91d32744028654ba7d19278a90, delay=966 ms 2024-11-24T03:55:16,946 DEBUG [StoreOpener-67da2bfde1dfe1c5a7387d8137ee444f-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/default/testBlockEvictionOnGracefulStop1732420514596/67da2bfde1dfe1c5a7387d8137ee444f/family/f6d06f91d32744028654ba7d19278a90 2024-11-24T03:55:16,947 INFO [StoreOpener-67da2bfde1dfe1c5a7387d8137ee444f-1 {}] regionserver.HStore(327): Store=67da2bfde1dfe1c5a7387d8137ee444f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T03:55:16,947 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 67da2bfde1dfe1c5a7387d8137ee444f 2024-11-24T03:55:16,948 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/default/testBlockEvictionOnGracefulStop1732420514596/67da2bfde1dfe1c5a7387d8137ee444f 2024-11-24T03:55:16,948 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/default/testBlockEvictionOnGracefulStop1732420514596/67da2bfde1dfe1c5a7387d8137ee444f 2024-11-24T03:55:16,948 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 67da2bfde1dfe1c5a7387d8137ee444f 2024-11-24T03:55:16,948 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 67da2bfde1dfe1c5a7387d8137ee444f 2024-11-24T03:55:16,949 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 67da2bfde1dfe1c5a7387d8137ee444f 2024-11-24T03:55:16,950 INFO [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 67da2bfde1dfe1c5a7387d8137ee444f; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58785968, jitterRate=-0.12402081489562988}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T03:55:16,950 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 67da2bfde1dfe1c5a7387d8137ee444f 2024-11-24T03:55:16,951 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 67da2bfde1dfe1c5a7387d8137ee444f: Running coprocessor pre-open hook at 1732420516942Writing region info on filesystem at 1732420516942Initializing all the Stores at 1732420516942Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732420516942Cleaning up temporary data from old regions at 1732420516948 (+6 ms)Running coprocessor post-open hooks at 1732420516950 (+2 ms)Region opened successfully at 1732420516951 (+1 ms) 2024-11-24T03:55:16,951 INFO [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732420514596,,1732420514600.67da2bfde1dfe1c5a7387d8137ee444f., pid=9, masterSystemTime=1732420516938 2024-11-24T03:55:16,954 DEBUG [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732420514596,,1732420514600.67da2bfde1dfe1c5a7387d8137ee444f. 2024-11-24T03:55:16,954 INFO [RS_OPEN_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732420514596,,1732420514600.67da2bfde1dfe1c5a7387d8137ee444f. 2024-11-24T03:55:16,954 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=67da2bfde1dfe1c5a7387d8137ee444f, regionState=OPEN, openSeqNum=10, regionLocation=71d8d2d6408d,33751,1732420513421 2024-11-24T03:55:16,956 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33121 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=71d8d2d6408d,33751,1732420513421, table=testBlockEvictionOnGracefulStop1732420514596, region=67da2bfde1dfe1c5a7387d8137ee444f. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T03:55:16,956 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 67da2bfde1dfe1c5a7387d8137ee444f, server=71d8d2d6408d,33751,1732420513421 because future has completed 2024-11-24T03:55:16,959 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-24T03:55:16,959 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 67da2bfde1dfe1c5a7387d8137ee444f, server=71d8d2d6408d,33751,1732420513421 in 173 msec 2024-11-24T03:55:16,961 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-24T03:55:16,961 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732420514596, region=67da2bfde1dfe1c5a7387d8137ee444f, ASSIGN in 332 msec 2024-11-24T03:55:16,961 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server 71d8d2d6408d,45285,1732420513461 after splitting done 2024-11-24T03:55:16,962 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 71d8d2d6408d,45285,1732420513461, splitWal=true, meta=false in 348 msec 2024-11-24T03:55:17,018 INFO [RS:2;71d8d2d6408d:39051 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T03:55:17,018 INFO [RS:2;71d8d2d6408d:39051 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=71d8d2d6408d%2C39051%2C1732420516797, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/WALs/71d8d2d6408d,39051,1732420516797, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/oldWALs, maxLogs=55 2024-11-24T03:55:17,035 INFO [RS:2;71d8d2d6408d:39051 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/WALs/71d8d2d6408d,39051,1732420516797/71d8d2d6408d%2C39051%2C1732420516797.1732420517019 2024-11-24T03:55:17,036 DEBUG [RS:2;71d8d2d6408d:39051 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T03:55:17,414 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-24T03:55:17,414 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T03:55:17,414 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T03:55:17,414 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T03:55:17,415 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T03:55:17,415 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T03:55:17,415 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-24T03:55:17,415 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=346014291, stopped=false 2024-11-24T03:55:17,415 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=71d8d2d6408d,33121,1732420513277 2024-11-24T03:55:17,454 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39051-0x1016c44c0b20003, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T03:55:17,454 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33121-0x1016c44c0b20000, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T03:55:17,454 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33751-0x1016c44c0b20001, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T03:55:17,454 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33121-0x1016c44c0b20000, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T03:55:17,454 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33751-0x1016c44c0b20001, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T03:55:17,454 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39051-0x1016c44c0b20003, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T03:55:17,454 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T03:55:17,455 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T03:55:17,456 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T03:55:17,456 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33121-0x1016c44c0b20000, quorum=127.0.0.1:49981, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T03:55:17,456 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T03:55:17,456 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39051-0x1016c44c0b20003, quorum=127.0.0.1:49981, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T03:55:17,457 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '71d8d2d6408d,33751,1732420513421' ***** 2024-11-24T03:55:17,457 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T03:55:17,457 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '71d8d2d6408d,39051,1732420516797' ***** 2024-11-24T03:55:17,457 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T03:55:17,457 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33751-0x1016c44c0b20001, quorum=127.0.0.1:49981, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T03:55:17,458 INFO [RS:0;71d8d2d6408d:33751 {}] bucket.BucketCache(1738): Disabling cache 2024-11-24T03:55:17,458 INFO [RS:2;71d8d2d6408d:39051 {}] bucket.BucketCache(1738): Disabling cache 2024-11-24T03:55:17,458 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-24T03:55:17,459 INFO [RS:0;71d8d2d6408d:33751 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9882d271-12f2-7101-b4a3-bbc86c31780c/bucket.persistence 2024-11-24T03:55:17,459 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-24T03:55:17,459 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-24T03:55:17,459 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-24T03:55:17,459 INFO [RS:2;71d8d2d6408d:39051 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9882d271-12f2-7101-b4a3-bbc86c31780c/bucket.persistence 2024-11-24T03:55:17,459 INFO [RS:0;71d8d2d6408d:33751 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-24T03:55:17,459 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-24T03:55:17,459 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-24T03:55:17,459 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-24T03:55:17,459 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-24T03:55:17,459 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-24T03:55:17,460 INFO [RS:2;71d8d2d6408d:39051 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-24T03:55:17,460 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-24T03:55:17,461 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1732420514596 2024-11-24T03:55:17,461 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1732420514596 Metrics about Tables on a single HBase RegionServer 2024-11-24T03:55:17,462 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T03:55:17,462 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-24T03:55:17,463 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-24T03:55:17,463 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-24T03:55:17,469 DEBUG [RS:0;71d8d2d6408d:33751 {}] bucket.BucketCache(1487): Thread RS:0;71d8d2d6408d:33751 started persisting bucket cache to file 2024-11-24T03:55:17,469 DEBUG [RS:0;71d8d2d6408d:33751 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T03:55:17,469 DEBUG [RS:0;71d8d2d6408d:33751 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T03:55:17,470 DEBUG [RS:2;71d8d2d6408d:39051 {}] bucket.BucketCache(1487): Thread RS:2;71d8d2d6408d:39051 started persisting bucket cache to file 2024-11-24T03:55:17,470 DEBUG [RS:2;71d8d2d6408d:39051 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T03:55:17,471 DEBUG [RS:2;71d8d2d6408d:39051 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T03:55:17,472 DEBUG [RS:0;71d8d2d6408d:33751 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-24T03:55:17,473 DEBUG [RS:0;71d8d2d6408d:33751 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732420517469 2024-11-24T03:55:17,473 DEBUG [RS:0;71d8d2d6408d:33751 {}] bucket.BucketCache(1509): Thread RS:0;71d8d2d6408d:33751 finished persisting bucket cache to file, renaming 2024-11-24T03:55:17,473 INFO [RS:0;71d8d2d6408d:33751 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T03:55:17,473 INFO [RS:0;71d8d2d6408d:33751 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T03:55:17,473 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T03:55:17,473 INFO [RS:0;71d8d2d6408d:33751 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T03:55:17,473 INFO [RS:0;71d8d2d6408d:33751 {}] regionserver.HRegionServer(3091): Received CLOSE for 67da2bfde1dfe1c5a7387d8137ee444f 2024-11-24T03:55:17,473 DEBUG [RS:2;71d8d2d6408d:39051 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-24T03:55:17,473 DEBUG [RS:2;71d8d2d6408d:39051 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732420517470 2024-11-24T03:55:17,474 DEBUG [RS:2;71d8d2d6408d:39051 {}] bucket.BucketCache(1509): Thread RS:2;71d8d2d6408d:39051 finished persisting bucket cache to file, renaming 2024-11-24T03:55:17,474 INFO [RS:0;71d8d2d6408d:33751 {}] regionserver.HRegionServer(959): stopping server 71d8d2d6408d,33751,1732420513421 2024-11-24T03:55:17,474 INFO [RS:0;71d8d2d6408d:33751 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T03:55:17,474 INFO [RS:0;71d8d2d6408d:33751 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;71d8d2d6408d:33751. 2024-11-24T03:55:17,474 INFO [RS:2;71d8d2d6408d:39051 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T03:55:17,474 INFO [RS_CLOSE_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732420514596,,1732420514600.67da2bfde1dfe1c5a7387d8137ee444f. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-24T03:55:17,474 DEBUG [RS:0;71d8d2d6408d:33751 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T03:55:17,474 INFO [RS:2;71d8d2d6408d:39051 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T03:55:17,474 DEBUG [RS:0;71d8d2d6408d:33751 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T03:55:17,474 DEBUG [RS_CLOSE_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 67da2bfde1dfe1c5a7387d8137ee444f, disabling compactions & flushes 2024-11-24T03:55:17,474 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T03:55:17,474 INFO [RS:2;71d8d2d6408d:39051 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T03:55:17,474 INFO [RS_CLOSE_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732420514596,,1732420514600.67da2bfde1dfe1c5a7387d8137ee444f. 2024-11-24T03:55:17,474 INFO [RS:2;71d8d2d6408d:39051 {}] regionserver.HRegionServer(959): stopping server 71d8d2d6408d,39051,1732420516797 2024-11-24T03:55:17,474 INFO [RS:0;71d8d2d6408d:33751 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T03:55:17,474 INFO [RS:2;71d8d2d6408d:39051 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T03:55:17,474 DEBUG [RS_CLOSE_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732420514596,,1732420514600.67da2bfde1dfe1c5a7387d8137ee444f. 2024-11-24T03:55:17,474 INFO [RS:0;71d8d2d6408d:33751 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T03:55:17,474 INFO [RS:2;71d8d2d6408d:39051 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;71d8d2d6408d:39051. 2024-11-24T03:55:17,474 INFO [RS:0;71d8d2d6408d:33751 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T03:55:17,474 DEBUG [RS_CLOSE_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732420514596,,1732420514600.67da2bfde1dfe1c5a7387d8137ee444f. after waiting 0 ms 2024-11-24T03:55:17,474 INFO [RS:0;71d8d2d6408d:33751 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-24T03:55:17,474 DEBUG [RS:2;71d8d2d6408d:39051 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T03:55:17,474 DEBUG [RS_CLOSE_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732420514596,,1732420514600.67da2bfde1dfe1c5a7387d8137ee444f. 2024-11-24T03:55:17,474 DEBUG [RS:2;71d8d2d6408d:39051 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T03:55:17,474 INFO [RS:2;71d8d2d6408d:39051 {}] regionserver.HRegionServer(976): stopping server 71d8d2d6408d,39051,1732420516797; all regions closed. 2024-11-24T03:55:17,475 INFO [RS:0;71d8d2d6408d:33751 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-24T03:55:17,475 DEBUG [RS:0;71d8d2d6408d:33751 {}] regionserver.HRegionServer(1325): Online Regions={67da2bfde1dfe1c5a7387d8137ee444f=testBlockEvictionOnGracefulStop1732420514596,,1732420514600.67da2bfde1dfe1c5a7387d8137ee444f., 1588230740=hbase:meta,,1.1588230740} 2024-11-24T03:55:17,475 DEBUG [RS:0;71d8d2d6408d:33751 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 67da2bfde1dfe1c5a7387d8137ee444f 2024-11-24T03:55:17,475 INFO [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-24T03:55:17,475 DEBUG [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T03:55:17,475 INFO [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T03:55:17,475 DEBUG [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T03:55:17,475 DEBUG [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T03:55:17,475 DEBUG [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T03:55:17,475 DEBUG [StoreFileCloser-67da2bfde1dfe1c5a7387d8137ee444f-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/default/testBlockEvictionOnGracefulStop1732420514596/67da2bfde1dfe1c5a7387d8137ee444f/family/f6d06f91d32744028654ba7d19278a90 2024-11-24T03:55:17,475 INFO [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.96 KB heapSize=5.58 KB 2024-11-24T03:55:17,479 DEBUG [RS:2;71d8d2d6408d:39051 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/oldWALs 2024-11-24T03:55:17,479 INFO [RS:2;71d8d2d6408d:39051 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 71d8d2d6408d%2C39051%2C1732420516797:(num 1732420517019) 2024-11-24T03:55:17,479 DEBUG [RS:2;71d8d2d6408d:39051 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T03:55:17,479 INFO [RS:2;71d8d2d6408d:39051 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T03:55:17,479 INFO [RS:2;71d8d2d6408d:39051 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T03:55:17,479 INFO [RS:2;71d8d2d6408d:39051 {}] hbase.ChoreService(370): Chore service for: regionserver/71d8d2d6408d:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-24T03:55:17,479 INFO [RS:2;71d8d2d6408d:39051 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T03:55:17,480 INFO [regionserver/71d8d2d6408d:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T03:55:17,480 INFO [RS:2;71d8d2d6408d:39051 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T03:55:17,480 INFO [RS:2;71d8d2d6408d:39051 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T03:55:17,480 INFO [RS:2;71d8d2d6408d:39051 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T03:55:17,480 DEBUG [RS_CLOSE_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/default/testBlockEvictionOnGracefulStop1732420514596/67da2bfde1dfe1c5a7387d8137ee444f/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-24T03:55:17,480 INFO [RS:2;71d8d2d6408d:39051 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39051 2024-11-24T03:55:17,480 INFO [RS_CLOSE_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732420514596,,1732420514600.67da2bfde1dfe1c5a7387d8137ee444f. 2024-11-24T03:55:17,480 DEBUG [RS_CLOSE_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 67da2bfde1dfe1c5a7387d8137ee444f: Waiting for close lock at 1732420517474Running coprocessor pre-close hooks at 1732420517474Disabling compacts and flushes for region at 1732420517474Disabling writes for close at 1732420517474Writing region close event to WAL at 1732420517475 (+1 ms)Running coprocessor post-close hooks at 1732420517480 (+5 ms)Closed at 1732420517480 2024-11-24T03:55:17,481 DEBUG [RS_CLOSE_REGION-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732420514596,,1732420514600.67da2bfde1dfe1c5a7387d8137ee444f. 2024-11-24T03:55:17,482 INFO [regionserver/71d8d2d6408d:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T03:55:17,484 DEBUG [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 0a45c60351cc4695975b070ad2192df9 2024-11-24T03:55:17,489 INFO [regionserver/71d8d2d6408d:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T03:55:17,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33121-0x1016c44c0b20000, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T03:55:17,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39051-0x1016c44c0b20003, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/71d8d2d6408d,39051,1732420516797 2024-11-24T03:55:17,498 INFO [RS:2;71d8d2d6408d:39051 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T03:55:17,498 DEBUG [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740/.tmp/info/0a45c60351cc4695975b070ad2192df9 is 209, key is testBlockEvictionOnGracefulStop1732420514596,,1732420514600.67da2bfde1dfe1c5a7387d8137ee444f./info:regioninfo/1732420516954/Put/seqid=0 2024-11-24T03:55:17,499 INFO [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.74 KB at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740/.tmp/info/0a45c60351cc4695975b070ad2192df9 2024-11-24T03:55:17,506 DEBUG [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 52a2b3f295b94f00842a39f56d11a91b 2024-11-24T03:55:17,512 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [71d8d2d6408d,39051,1732420516797] 2024-11-24T03:55:17,518 DEBUG [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740/.tmp/ns/52a2b3f295b94f00842a39f56d11a91b is 43, key is default/ns:d/1732420514539/Put/seqid=0 2024-11-24T03:55:17,518 INFO [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740/.tmp/ns/52a2b3f295b94f00842a39f56d11a91b 2024-11-24T03:55:17,520 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/71d8d2d6408d,39051,1732420516797 already deleted, retry=false 2024-11-24T03:55:17,520 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 71d8d2d6408d,39051,1732420516797 expired; onlineServers=1 2024-11-24T03:55:17,526 DEBUG [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 3344dc7374d147548e7bbb3107487fec 2024-11-24T03:55:17,538 DEBUG [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740/.tmp/table/3344dc7374d147548e7bbb3107487fec is 80, key is testBlockEvictionOnGracefulStop1732420514596/table:state/1732420514982/Put/seqid=0 2024-11-24T03:55:17,539 INFO [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740/.tmp/table/3344dc7374d147548e7bbb3107487fec 2024-11-24T03:55:17,541 DEBUG [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740/.tmp/info/0a45c60351cc4695975b070ad2192df9 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740/info/0a45c60351cc4695975b070ad2192df9 2024-11-24T03:55:17,542 DEBUG [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740/info/0a45c60351cc4695975b070ad2192df9, delay=932 ms 2024-11-24T03:55:17,543 INFO [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740/info/0a45c60351cc4695975b070ad2192df9, entries=14, sequenceid=13, filesize=7.5 K 2024-11-24T03:55:17,544 DEBUG [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740/.tmp/ns/52a2b3f295b94f00842a39f56d11a91b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740/ns/52a2b3f295b94f00842a39f56d11a91b 2024-11-24T03:55:17,544 DEBUG [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740/ns/52a2b3f295b94f00842a39f56d11a91b, delay=926 ms 2024-11-24T03:55:17,545 INFO [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740/ns/52a2b3f295b94f00842a39f56d11a91b, entries=2, sequenceid=13, filesize=5.0 K 2024-11-24T03:55:17,546 DEBUG [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740/.tmp/table/3344dc7374d147548e7bbb3107487fec as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740/table/3344dc7374d147548e7bbb3107487fec 2024-11-24T03:55:17,547 DEBUG [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740/table/3344dc7374d147548e7bbb3107487fec, delay=911 ms 2024-11-24T03:55:17,548 INFO [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740/table/3344dc7374d147548e7bbb3107487fec, entries=2, sequenceid=13, filesize=5.3 K 2024-11-24T03:55:17,548 INFO [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 73ms, sequenceid=13, compaction requested=false 2024-11-24T03:55:17,549 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740/info/0a45c60351cc4695975b070ad2192df9 2024-11-24T03:55:17,550 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740/ns/52a2b3f295b94f00842a39f56d11a91b 2024-11-24T03:55:17,550 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740/table/3344dc7374d147548e7bbb3107487fec 2024-11-24T03:55:17,555 DEBUG [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/data/hbase/meta/1588230740/recovered.edits/16.seqid, newMaxSeqId=16, maxSeqId=1 2024-11-24T03:55:17,555 DEBUG [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T03:55:17,555 INFO [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T03:55:17,555 DEBUG [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732420517475Running coprocessor pre-close hooks at 1732420517475Disabling compacts and flushes for region at 1732420517475Disabling writes for close at 1732420517475Obtaining lock to block concurrent updates at 1732420517475Preparing flush snapshotting stores in 1588230740 at 1732420517475Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3035, getHeapSize=5648, getOffHeapSize=0, getCellsCount=22 at 1732420517476 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732420517476Flushing 1588230740/info: creating writer at 1732420517476Flushing 1588230740/info: appending metadata at 1732420517497 (+21 ms)Flushing 1588230740/info: closing flushed file at 1732420517497Flushing 1588230740/ns: creating writer at 1732420517501 (+4 ms)Flushing 1588230740/ns: appending metadata at 1732420517517 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1732420517517Flushing 1588230740/table: creating writer at 1732420517520 (+3 ms)Flushing 1588230740/table: appending metadata at 1732420517537 (+17 ms)Flushing 1588230740/table: closing flushed file at 1732420517537Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69bd688b: reopening flushed file at 1732420517541 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e67fff9: reopening flushed file at 1732420517543 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5ffcfe1c: reopening flushed file at 1732420517546 (+3 ms)Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 73ms, sequenceid=13, compaction requested=false at 1732420517548 (+2 ms)Writing region close event to WAL at 1732420517551 (+3 ms)Running coprocessor post-close hooks at 1732420517555 (+4 ms)Closed at 1732420517555 2024-11-24T03:55:17,555 DEBUG [RS_CLOSE_META-regionserver/71d8d2d6408d:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-24T03:55:17,612 INFO [RS:2;71d8d2d6408d:39051 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T03:55:17,612 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39051-0x1016c44c0b20003, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T03:55:17,612 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39051-0x1016c44c0b20003, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T03:55:17,612 INFO [RS:2;71d8d2d6408d:39051 {}] regionserver.HRegionServer(1031): Exiting; stopping=71d8d2d6408d,39051,1732420516797; zookeeper connection closed. 2024-11-24T03:55:17,613 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4a6da04f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4a6da04f 2024-11-24T03:55:17,675 INFO [RS:0;71d8d2d6408d:33751 {}] regionserver.HRegionServer(976): stopping server 71d8d2d6408d,33751,1732420513421; all regions closed. 2024-11-24T03:55:17,679 DEBUG [RS:0;71d8d2d6408d:33751 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/oldWALs 2024-11-24T03:55:17,679 INFO [RS:0;71d8d2d6408d:33751 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 71d8d2d6408d%2C33751%2C1732420513421.meta:.meta(num 1732420514467) 2024-11-24T03:55:17,682 DEBUG [RS:0;71d8d2d6408d:33751 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/oldWALs 2024-11-24T03:55:17,682 INFO [RS:0;71d8d2d6408d:33751 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 71d8d2d6408d%2C33751%2C1732420513421:(num 1732420514120) 2024-11-24T03:55:17,682 DEBUG [RS:0;71d8d2d6408d:33751 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T03:55:17,682 INFO [RS:0;71d8d2d6408d:33751 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T03:55:17,682 INFO [RS:0;71d8d2d6408d:33751 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T03:55:17,683 INFO [RS:0;71d8d2d6408d:33751 {}] hbase.ChoreService(370): Chore service for: regionserver/71d8d2d6408d:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T03:55:17,683 INFO [RS:0;71d8d2d6408d:33751 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T03:55:17,683 INFO [regionserver/71d8d2d6408d:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T03:55:17,683 INFO [RS:0;71d8d2d6408d:33751 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33751 2024-11-24T03:55:17,703 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33751-0x1016c44c0b20001, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/71d8d2d6408d,33751,1732420513421 2024-11-24T03:55:17,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33121-0x1016c44c0b20000, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T03:55:17,704 INFO [RS:0;71d8d2d6408d:33751 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T03:55:17,712 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [71d8d2d6408d,33751,1732420513421] 2024-11-24T03:55:17,720 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/71d8d2d6408d,33751,1732420513421 already deleted, retry=false 2024-11-24T03:55:17,720 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 71d8d2d6408d,33751,1732420513421 expired; onlineServers=0 2024-11-24T03:55:17,720 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '71d8d2d6408d,33121,1732420513277' ***** 2024-11-24T03:55:17,720 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-24T03:55:17,720 INFO [M:0;71d8d2d6408d:33121 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T03:55:17,720 INFO [M:0;71d8d2d6408d:33121 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T03:55:17,720 DEBUG [M:0;71d8d2d6408d:33121 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-24T03:55:17,720 DEBUG [M:0;71d8d2d6408d:33121 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-24T03:55:17,720 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-24T03:55:17,720 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster-HFileCleaner.small.0-1732420513816 {}] cleaner.HFileCleaner(306): Exit Thread[master/71d8d2d6408d:0:becomeActiveMaster-HFileCleaner.small.0-1732420513816,5,FailOnTimeoutGroup] 2024-11-24T03:55:17,720 DEBUG [master/71d8d2d6408d:0:becomeActiveMaster-HFileCleaner.large.0-1732420513815 {}] cleaner.HFileCleaner(306): Exit Thread[master/71d8d2d6408d:0:becomeActiveMaster-HFileCleaner.large.0-1732420513815,5,FailOnTimeoutGroup] 2024-11-24T03:55:17,721 INFO [M:0;71d8d2d6408d:33121 {}] hbase.ChoreService(370): Chore service for: master/71d8d2d6408d:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-24T03:55:17,721 INFO [M:0;71d8d2d6408d:33121 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T03:55:17,721 DEBUG [M:0;71d8d2d6408d:33121 {}] master.HMaster(1795): Stopping service threads 2024-11-24T03:55:17,721 INFO [M:0;71d8d2d6408d:33121 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-24T03:55:17,721 INFO [M:0;71d8d2d6408d:33121 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T03:55:17,721 INFO [M:0;71d8d2d6408d:33121 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T03:55:17,722 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-24T03:55:17,728 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33121-0x1016c44c0b20000, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-24T03:55:17,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33121-0x1016c44c0b20000, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T03:55:17,729 DEBUG [M:0;71d8d2d6408d:33121 {}] zookeeper.ZKUtil(347): master:33121-0x1016c44c0b20000, quorum=127.0.0.1:49981, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-24T03:55:17,729 WARN [M:0;71d8d2d6408d:33121 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-24T03:55:17,729 INFO [M:0;71d8d2d6408d:33121 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/.lastflushedseqids 2024-11-24T03:55:17,736 INFO [M:0;71d8d2d6408d:33121 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T03:55:17,736 INFO [M:0;71d8d2d6408d:33121 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-24T03:55:17,736 DEBUG [M:0;71d8d2d6408d:33121 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T03:55:17,736 INFO [M:0;71d8d2d6408d:33121 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T03:55:17,736 DEBUG [M:0;71d8d2d6408d:33121 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T03:55:17,736 DEBUG [M:0;71d8d2d6408d:33121 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T03:55:17,736 DEBUG [M:0;71d8d2d6408d:33121 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T03:55:17,736 INFO [M:0;71d8d2d6408d:33121 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=32.81 KB heapSize=41.05 KB 2024-11-24T03:55:17,745 DEBUG [M:0;71d8d2d6408d:33121 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 569c4748076644fab7e4345e31e7e883 2024-11-24T03:55:17,758 DEBUG [M:0;71d8d2d6408d:33121 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/569c4748076644fab7e4345e31e7e883 is 82, key is hbase:meta,,1/info:regioninfo/1732420514498/Put/seqid=0 2024-11-24T03:55:17,758 INFO [M:0;71d8d2d6408d:33121 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/569c4748076644fab7e4345e31e7e883 2024-11-24T03:55:17,765 DEBUG [M:0;71d8d2d6408d:33121 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c6a2f90e96d14de1ba759daeec7c4220 2024-11-24T03:55:17,778 DEBUG [M:0;71d8d2d6408d:33121 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c6a2f90e96d14de1ba759daeec7c4220 is 781, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732420514986/Put/seqid=0 2024-11-24T03:55:17,779 INFO [M:0;71d8d2d6408d:33121 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.08 KB at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c6a2f90e96d14de1ba759daeec7c4220 2024-11-24T03:55:17,786 DEBUG [M:0;71d8d2d6408d:33121 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 57565d89c0124a868b3abcf5b20654a1 2024-11-24T03:55:17,798 DEBUG [M:0;71d8d2d6408d:33121 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/57565d89c0124a868b3abcf5b20654a1 is 69, key is 71d8d2d6408d,33751,1732420513421/rs:state/1732420513932/Put/seqid=0 2024-11-24T03:55:17,799 INFO [M:0;71d8d2d6408d:33121 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/57565d89c0124a868b3abcf5b20654a1 2024-11-24T03:55:17,801 INFO [M:0;71d8d2d6408d:33121 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 57565d89c0124a868b3abcf5b20654a1 2024-11-24T03:55:17,801 DEBUG [M:0;71d8d2d6408d:33121 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/569c4748076644fab7e4345e31e7e883 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/569c4748076644fab7e4345e31e7e883 2024-11-24T03:55:17,803 INFO [M:0;71d8d2d6408d:33121 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/569c4748076644fab7e4345e31e7e883, entries=8, sequenceid=83, filesize=5.5 K 2024-11-24T03:55:17,803 DEBUG [M:0;71d8d2d6408d:33121 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c6a2f90e96d14de1ba759daeec7c4220 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c6a2f90e96d14de1ba759daeec7c4220 2024-11-24T03:55:17,804 INFO [M:0;71d8d2d6408d:33121 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c6a2f90e96d14de1ba759daeec7c4220, entries=9, sequenceid=83, filesize=6.4 K 2024-11-24T03:55:17,805 DEBUG [M:0;71d8d2d6408d:33121 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/57565d89c0124a868b3abcf5b20654a1 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/57565d89c0124a868b3abcf5b20654a1 2024-11-24T03:55:17,806 INFO [M:0;71d8d2d6408d:33121 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 57565d89c0124a868b3abcf5b20654a1 2024-11-24T03:55:17,806 INFO [M:0;71d8d2d6408d:33121 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7e33209f-f4a0-5468-84ad-1d4b11830a17/7f166d94-9d3e-4310-d20e-4640b5c6de7b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/57565d89c0124a868b3abcf5b20654a1, entries=3, sequenceid=83, filesize=5.3 K 2024-11-24T03:55:17,807 INFO [M:0;71d8d2d6408d:33121 {}] regionserver.HRegion(3140): Finished flush of dataSize ~32.81 KB/33602, heapSize ~40.76 KB/41736, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 71ms, sequenceid=83, compaction requested=false 2024-11-24T03:55:17,809 INFO [M:0;71d8d2d6408d:33121 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T03:55:17,809 DEBUG [M:0;71d8d2d6408d:33121 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732420517736Disabling compacts and flushes for region at 1732420517736Disabling writes for close at 1732420517736Obtaining lock to block concurrent updates at 1732420517736Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732420517736Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=33602, getHeapSize=41976, getOffHeapSize=0, getCellsCount=98 at 1732420517737 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732420517737Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732420517737Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732420517757 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732420517757Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732420517760 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732420517778 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732420517778Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732420517781 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732420517798 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732420517798Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@373fcd16: reopening flushed file at 1732420517801 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ae97ec8: reopening flushed file at 1732420517803 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51942766: reopening flushed file at 1732420517804 (+1 ms)Finished flush of dataSize ~32.81 KB/33602, heapSize ~40.76 KB/41736, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 71ms, sequenceid=83, compaction requested=false at 1732420517807 (+3 ms)Writing region close event to WAL at 1732420517809 (+2 ms)Closed at 1732420517809 2024-11-24T03:55:17,810 INFO [M:0;71d8d2d6408d:33121 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-24T03:55:17,810 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T03:55:17,810 INFO [M:0;71d8d2d6408d:33121 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33121 2024-11-24T03:55:17,810 INFO [M:0;71d8d2d6408d:33121 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T03:55:17,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33751-0x1016c44c0b20001, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T03:55:17,812 INFO [RS:0;71d8d2d6408d:33751 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T03:55:17,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33751-0x1016c44c0b20001, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T03:55:17,812 INFO [RS:0;71d8d2d6408d:33751 {}] regionserver.HRegionServer(1031): Exiting; stopping=71d8d2d6408d,33751,1732420513421; zookeeper connection closed. 2024-11-24T03:55:17,812 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@b52a08e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@b52a08e 2024-11-24T03:55:17,812 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-24T03:55:17,920 INFO [M:0;71d8d2d6408d:33121 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T03:55:17,920 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33121-0x1016c44c0b20000, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T03:55:17,920 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33121-0x1016c44c0b20000, quorum=127.0.0.1:49981, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T03:55:17,922 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-24T03:55:17,923 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-24T03:55:17,924 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T03:55:17,929 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=72 (was 51) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=240 (was 244), ProcessCount=11 (was 11), AvailableMemoryMB=7732 (was 7842)