2024-12-09 03:45:12,135 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e 2024-12-09 03:45:12,148 main DEBUG Took 0.010981 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-09 03:45:12,148 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-09 03:45:12,148 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-09 03:45:12,149 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-09 03:45:12,150 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 03:45:12,158 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-09 03:45:12,178 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 03:45:12,180 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 03:45:12,181 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 03:45:12,181 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 03:45:12,182 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 03:45:12,182 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 03:45:12,184 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 03:45:12,184 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 03:45:12,185 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 03:45:12,185 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 03:45:12,186 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 03:45:12,187 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 03:45:12,187 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 03:45:12,188 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 03:45:12,189 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 03:45:12,189 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 03:45:12,190 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 03:45:12,190 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 03:45:12,191 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 03:45:12,191 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 03:45:12,192 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 03:45:12,192 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 03:45:12,193 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 03:45:12,193 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 03:45:12,194 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 03:45:12,194 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-09 03:45:12,196 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 03:45:12,198 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-09 03:45:12,200 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-09 03:45:12,201 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-09 03:45:12,203 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-09 03:45:12,203 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-09 03:45:12,216 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-09 03:45:12,220 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-09 03:45:12,222 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-09 03:45:12,222 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-09 03:45:12,223 main DEBUG createAppenders(={Console}) 2024-12-09 03:45:12,224 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e initialized 2024-12-09 03:45:12,224 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e 2024-12-09 03:45:12,225 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e OK. 2024-12-09 03:45:12,226 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-09 03:45:12,226 main DEBUG OutputStream closed 2024-12-09 03:45:12,226 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-09 03:45:12,227 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-09 03:45:12,227 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@74751b3 OK 2024-12-09 03:45:12,309 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-09 03:45:12,311 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-09 03:45:12,312 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-09 03:45:12,313 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-09 03:45:12,314 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-09 03:45:12,314 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-09 03:45:12,314 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-09 03:45:12,315 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-09 03:45:12,315 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-09 03:45:12,315 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-09 03:45:12,315 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-09 03:45:12,316 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-09 03:45:12,316 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-09 03:45:12,316 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-09 03:45:12,317 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-09 03:45:12,317 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-09 03:45:12,317 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-09 03:45:12,318 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-09 03:45:12,320 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-09 03:45:12,321 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@528c868) with optional ClassLoader: null 2024-12-09 03:45:12,321 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-09 03:45:12,322 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@528c868] started OK. 2024-12-09T03:45:12,337 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestPrefetchRSClose timeout: 13 mins 2024-12-09 03:45:12,339 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-09 03:45:12,340 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-09T03:45:12,605 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c 2024-12-09T03:45:12,633 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestPrefetchRSClose#testPrefetchPersistence Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=907, ProcessCount=11, AvailableMemoryMB=12717 2024-12-09T03:45:12,733 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-09T03:45:12,829 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T03:45:12,846 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/cluster_ade5d35a-9cfb-a04b-1ce0-6d9de508891e, deleteOnExit=true 2024-12-09T03:45:12,938 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/cluster_ade5d35a-9cfb-a04b-1ce0-6d9de508891e/zookeeper_0, clientPort=60363, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/cluster_ade5d35a-9cfb-a04b-1ce0-6d9de508891e/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/cluster_ade5d35a-9cfb-a04b-1ce0-6d9de508891e/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-09T03:45:12,953 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60363 2024-12-09T03:45:12,953 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T03:45:13,039 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b with version=8 2024-12-09T03:45:13,039 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/hbase-staging 2024-12-09T03:45:13,142 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-09T03:45:13,416 INFO [Time-limited test {}] client.ConnectionUtils(128): master/55d0183f16d2:0 server-side Connection retries=45 2024-12-09T03:45:13,427 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T03:45:13,429 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T03:45:13,436 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T03:45:13,437 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T03:45:13,437 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T03:45:13,576 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-09T03:45:13,667 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T03:45:13,711 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-09T03:45:13,723 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-09T03:45:13,727 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T03:45:13,771 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 14953 (auto-detected) 2024-12-09T03:45:13,772 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-09T03:45:13,792 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39199 2024-12-09T03:45:13,815 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39199 connecting to ZooKeeper ensemble=127.0.0.1:60363 2024-12-09T03:45:13,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:391990x0, quorum=127.0.0.1:60363, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:45:13,856 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39199-0x1019553535e0000 connected 2024-12-09T03:45:13,884 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T03:45:13,885 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T03:45:13,893 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39199-0x1019553535e0000, quorum=127.0.0.1:60363, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T03:45:13,897 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b, hbase.cluster.distributed=false 2024-12-09T03:45:13,924 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39199-0x1019553535e0000, quorum=127.0.0.1:60363, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T03:45:13,929 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39199 2024-12-09T03:45:13,930 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39199 2024-12-09T03:45:13,930 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39199 2024-12-09T03:45:13,931 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39199 2024-12-09T03:45:13,931 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39199 2024-12-09T03:45:14,020 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/55d0183f16d2:0 server-side Connection retries=45 2024-12-09T03:45:14,022 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T03:45:14,023 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T03:45:14,023 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T03:45:14,023 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T03:45:14,023 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T03:45:14,026 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-09T03:45:14,029 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T03:45:14,030 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40391 2024-12-09T03:45:14,032 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40391 connecting to ZooKeeper ensemble=127.0.0.1:60363 2024-12-09T03:45:14,032 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T03:45:14,033 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T03:45:14,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:403910x0, quorum=127.0.0.1:60363, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:45:14,038 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40391-0x1019553535e0001 connected 2024-12-09T03:45:14,038 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40391-0x1019553535e0001, quorum=127.0.0.1:60363, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T03:45:14,044 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-09T03:45:14,056 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/bucket.cache 2024-12-09T03:45:14,062 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-09T03:45:14,064 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-09T03:45:14,065 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 1000 2024-12-09T03:45:14,067 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-09T03:45:14,067 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-09T03:45:14,075 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-09T03:45:14,076 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-09T03:45:14,076 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/bucket.persistence is complete. 2024-12-09T03:45:14,082 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-09T03:45:14,086 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40391-0x1019553535e0001, quorum=127.0.0.1:60363, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-09T03:45:14,092 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40391-0x1019553535e0001, quorum=127.0.0.1:60363, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T03:45:14,093 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40391 2024-12-09T03:45:14,094 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40391 2024-12-09T03:45:14,094 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40391 2024-12-09T03:45:14,096 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40391 2024-12-09T03:45:14,100 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40391 2024-12-09T03:45:14,124 DEBUG [M:0;55d0183f16d2:39199 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;55d0183f16d2:39199 2024-12-09T03:45:14,125 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/55d0183f16d2,39199,1733715913214 2024-12-09T03:45:14,133 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39199-0x1019553535e0000, quorum=127.0.0.1:60363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T03:45:14,133 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40391-0x1019553535e0001, quorum=127.0.0.1:60363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T03:45:14,135 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39199-0x1019553535e0000, quorum=127.0.0.1:60363, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/55d0183f16d2,39199,1733715913214 2024-12-09T03:45:14,169 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39199-0x1019553535e0000, quorum=127.0.0.1:60363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:45:14,169 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40391-0x1019553535e0001, quorum=127.0.0.1:60363, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-09T03:45:14,170 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40391-0x1019553535e0001, quorum=127.0.0.1:60363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:45:14,171 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39199-0x1019553535e0000, quorum=127.0.0.1:60363, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-09T03:45:14,172 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/55d0183f16d2,39199,1733715913214 from backup master directory 2024-12-09T03:45:14,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40391-0x1019553535e0001, quorum=127.0.0.1:60363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T03:45:14,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39199-0x1019553535e0000, quorum=127.0.0.1:60363, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/55d0183f16d2,39199,1733715913214 2024-12-09T03:45:14,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39199-0x1019553535e0000, quorum=127.0.0.1:60363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T03:45:14,176 WARN [master/55d0183f16d2:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T03:45:14,177 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=55d0183f16d2,39199,1733715913214 2024-12-09T03:45:14,180 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-09T03:45:14,182 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-09T03:45:14,193 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/hbase.id] with ID: 167d8e73-c81d-4579-b088-40d1926afeae 2024-12-09T03:45:14,194 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/.tmp/hbase.id 2024-12-09T03:45:14,200 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/hbase.id] 2024-12-09T03:45:14,245 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T03:45:14,250 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-09T03:45:14,271 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 19ms. 2024-12-09T03:45:14,275 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40391-0x1019553535e0001, quorum=127.0.0.1:60363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:45:14,275 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39199-0x1019553535e0000, quorum=127.0.0.1:60363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:45:14,310 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T03:45:14,314 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-09T03:45:14,364 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:45:14,370 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T03:45:14,431 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/MasterData/data/master/store 2024-12-09T03:45:14,451 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-09T03:45:14,455 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T03:45:14,456 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-09T03:45:14,457 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T03:45:14,457 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T03:45:14,458 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-09T03:45:14,459 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T03:45:14,459 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T03:45:14,459 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733715914456Disabling compacts and flushes for region at 1733715914456Disabling writes for close at 1733715914458 (+2 ms)Writing region close event to WAL at 1733715914459 (+1 ms)Closed at 1733715914459 2024-12-09T03:45:14,460 WARN [master/55d0183f16d2:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/MasterData/data/master/store/.initializing 2024-12-09T03:45:14,460 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/MasterData/WALs/55d0183f16d2,39199,1733715913214 2024-12-09T03:45:14,474 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T03:45:14,488 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=55d0183f16d2%2C39199%2C1733715913214, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/MasterData/WALs/55d0183f16d2,39199,1733715913214, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/MasterData/oldWALs, maxLogs=10 2024-12-09T03:45:14,532 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/MasterData/WALs/55d0183f16d2,39199,1733715913214/55d0183f16d2%2C39199%2C1733715913214.1733715914493 2024-12-09T03:45:14,533 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-09T03:45:14,534 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-09T03:45:14,535 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T03:45:14,540 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T03:45:14,541 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T03:45:14,586 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T03:45:14,618 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-09T03:45:14,623 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:45:14,625 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T03:45:14,626 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T03:45:14,632 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-09T03:45:14,632 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:45:14,633 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:45:14,633 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T03:45:14,639 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-09T03:45:14,639 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:45:14,640 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:45:14,640 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T03:45:14,644 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-09T03:45:14,645 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:45:14,645 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:45:14,646 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T03:45:14,650 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-09T03:45:14,651 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-09T03:45:14,656 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T03:45:14,657 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T03:45:14,661 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T03:45:14,664 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T03:45:14,673 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T03:45:14,674 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60679382, jitterRate=-0.09580674767494202}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T03:45:14,682 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733715914560Initializing all the Stores at 1733715914560Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733715914561 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733715914562 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733715914562Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733715914563 (+1 ms)Cleaning up temporary data from old regions at 1733715914657 (+94 ms)Region opened successfully at 1733715914682 (+25 ms) 2024-12-09T03:45:14,684 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-09T03:45:14,727 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@406d76a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=55d0183f16d2/172.17.0.2:0 2024-12-09T03:45:14,771 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-09T03:45:14,783 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-09T03:45:14,783 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-09T03:45:14,786 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-09T03:45:14,787 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-09T03:45:14,793 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-09T03:45:14,793 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-09T03:45:14,818 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-09T03:45:14,828 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39199-0x1019553535e0000, quorum=127.0.0.1:60363, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-09T03:45:14,830 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-09T03:45:14,833 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-09T03:45:14,834 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39199-0x1019553535e0000, quorum=127.0.0.1:60363, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-09T03:45:14,837 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-09T03:45:14,839 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-09T03:45:14,843 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39199-0x1019553535e0000, quorum=127.0.0.1:60363, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-09T03:45:14,845 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-09T03:45:14,847 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39199-0x1019553535e0000, quorum=127.0.0.1:60363, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-09T03:45:14,848 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-09T03:45:14,869 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39199-0x1019553535e0000, quorum=127.0.0.1:60363, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-09T03:45:14,871 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-09T03:45:14,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39199-0x1019553535e0000, quorum=127.0.0.1:60363, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T03:45:14,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40391-0x1019553535e0001, quorum=127.0.0.1:60363, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T03:45:14,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40391-0x1019553535e0001, quorum=127.0.0.1:60363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:45:14,889 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39199-0x1019553535e0000, quorum=127.0.0.1:60363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:45:14,893 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=55d0183f16d2,39199,1733715913214, sessionid=0x1019553535e0000, setting cluster-up flag (Was=false) 2024-12-09T03:45:14,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39199-0x1019553535e0000, quorum=127.0.0.1:60363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:45:14,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40391-0x1019553535e0001, quorum=127.0.0.1:60363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:45:14,916 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-09T03:45:14,918 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=55d0183f16d2,39199,1733715913214 2024-12-09T03:45:14,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40391-0x1019553535e0001, quorum=127.0.0.1:60363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:45:14,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39199-0x1019553535e0000, quorum=127.0.0.1:60363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:45:14,930 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-09T03:45:14,932 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=55d0183f16d2,39199,1733715913214 2024-12-09T03:45:14,938 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-09T03:45:15,005 INFO [RS:0;55d0183f16d2:40391 {}] regionserver.HRegionServer(746): ClusterId : 167d8e73-c81d-4579-b088-40d1926afeae 2024-12-09T03:45:15,008 DEBUG [RS:0;55d0183f16d2:40391 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-09T03:45:15,013 DEBUG [RS:0;55d0183f16d2:40391 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-09T03:45:15,013 DEBUG [RS:0;55d0183f16d2:40391 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-09T03:45:15,018 DEBUG [RS:0;55d0183f16d2:40391 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-09T03:45:15,019 DEBUG [RS:0;55d0183f16d2:40391 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5df7ed03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=55d0183f16d2/172.17.0.2:0 2024-12-09T03:45:15,022 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-09T03:45:15,033 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-09T03:45:15,036 DEBUG [RS:0;55d0183f16d2:40391 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;55d0183f16d2:40391 2024-12-09T03:45:15,039 INFO [RS:0;55d0183f16d2:40391 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-09T03:45:15,039 INFO [RS:0;55d0183f16d2:40391 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-09T03:45:15,040 DEBUG [RS:0;55d0183f16d2:40391 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-09T03:45:15,043 INFO [RS:0;55d0183f16d2:40391 {}] regionserver.HRegionServer(2659): reportForDuty to master=55d0183f16d2,39199,1733715913214 with port=40391, startcode=1733715913981 2024-12-09T03:45:15,043 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-09T03:45:15,049 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 55d0183f16d2,39199,1733715913214 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-09T03:45:15,057 DEBUG [RS:0;55d0183f16d2:40391 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-09T03:45:15,057 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/55d0183f16d2:0, corePoolSize=5, maxPoolSize=5 2024-12-09T03:45:15,057 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/55d0183f16d2:0, corePoolSize=5, maxPoolSize=5 2024-12-09T03:45:15,058 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/55d0183f16d2:0, corePoolSize=5, maxPoolSize=5 2024-12-09T03:45:15,058 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/55d0183f16d2:0, corePoolSize=5, maxPoolSize=5 2024-12-09T03:45:15,058 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/55d0183f16d2:0, corePoolSize=10, maxPoolSize=10 2024-12-09T03:45:15,058 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/55d0183f16d2:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:45:15,058 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/55d0183f16d2:0, corePoolSize=2, maxPoolSize=2 2024-12-09T03:45:15,059 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/55d0183f16d2:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:45:15,060 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733715945060 2024-12-09T03:45:15,062 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-09T03:45:15,063 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T03:45:15,063 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-09T03:45:15,064 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-09T03:45:15,067 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:45:15,067 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-09T03:45:15,067 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-09T03:45:15,068 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-09T03:45:15,068 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-09T03:45:15,068 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-09T03:45:15,069 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-09T03:45:15,073 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-09T03:45:15,075 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-09T03:45:15,075 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-09T03:45:15,077 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-09T03:45:15,077 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-09T03:45:15,079 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/55d0183f16d2:0:becomeActiveMaster-HFileCleaner.large.0-1733715915078,5,FailOnTimeoutGroup] 2024-12-09T03:45:15,080 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/55d0183f16d2:0:becomeActiveMaster-HFileCleaner.small.0-1733715915079,5,FailOnTimeoutGroup] 2024-12-09T03:45:15,080 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-09T03:45:15,081 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-09T03:45:15,082 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-09T03:45:15,083 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-09T03:45:15,083 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-09T03:45:15,083 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b 2024-12-09T03:45:15,095 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T03:45:15,096 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-09T03:45:15,103 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T03:45:15,103 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:45:15,104 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T03:45:15,104 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-09T03:45:15,109 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-09T03:45:15,110 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:45:15,110 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T03:45:15,110 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-09T03:45:15,119 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T03:45:15,120 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:45:15,120 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T03:45:15,120 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-09T03:45:15,125 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T03:45:15,125 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:45:15,125 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T03:45:15,126 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-09T03:45:15,128 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/data/hbase/meta/1588230740 2024-12-09T03:45:15,128 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/data/hbase/meta/1588230740 2024-12-09T03:45:15,129 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-09T03:45:15,129 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-09T03:45:15,129 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57623, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-09T03:45:15,129 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T03:45:15,130 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-09T03:45:15,137 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39199 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 55d0183f16d2,40391,1733715913981 2024-12-09T03:45:15,140 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39199 {}] master.ServerManager(517): Registering regionserver=55d0183f16d2,40391,1733715913981 2024-12-09T03:45:15,143 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T03:45:15,143 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59141834, jitterRate=-0.11871799826622009}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T03:45:15,147 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733715915095Initializing all the Stores at 1733715915096 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733715915096Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733715915096Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733715915096Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733715915096Cleaning up temporary data from old regions at 1733715915129 (+33 ms)Region opened successfully at 1733715915147 (+18 ms) 2024-12-09T03:45:15,148 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-09T03:45:15,148 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-09T03:45:15,148 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-09T03:45:15,148 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T03:45:15,148 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T03:45:15,149 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-09T03:45:15,150 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733715915147Disabling compacts and flushes for region at 1733715915147Disabling writes for close at 1733715915148 (+1 ms)Writing region close event to WAL at 1733715915149 (+1 ms)Closed at 1733715915149 2024-12-09T03:45:15,152 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T03:45:15,152 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-09T03:45:15,156 DEBUG [RS:0;55d0183f16d2:40391 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b 2024-12-09T03:45:15,156 DEBUG [RS:0;55d0183f16d2:40391 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-09T03:45:15,156 DEBUG [RS:0;55d0183f16d2:40391 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-09T03:45:15,160 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-09T03:45:15,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39199-0x1019553535e0000, quorum=127.0.0.1:60363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T03:45:15,161 DEBUG [RS:0;55d0183f16d2:40391 {}] zookeeper.ZKUtil(111): regionserver:40391-0x1019553535e0001, quorum=127.0.0.1:60363, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/55d0183f16d2,40391,1733715913981 2024-12-09T03:45:15,161 WARN [RS:0;55d0183f16d2:40391 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T03:45:15,162 INFO [RS:0;55d0183f16d2:40391 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T03:45:15,162 DEBUG [RS:0;55d0183f16d2:40391 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/WALs/55d0183f16d2,40391,1733715913981 2024-12-09T03:45:15,164 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [55d0183f16d2,40391,1733715913981] 2024-12-09T03:45:15,168 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T03:45:15,170 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-09T03:45:15,189 INFO [RS:0;55d0183f16d2:40391 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-09T03:45:15,202 INFO [RS:0;55d0183f16d2:40391 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-09T03:45:15,207 INFO [RS:0;55d0183f16d2:40391 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T03:45:15,207 INFO [RS:0;55d0183f16d2:40391 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T03:45:15,208 INFO [RS:0;55d0183f16d2:40391 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-09T03:45:15,213 INFO [RS:0;55d0183f16d2:40391 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-09T03:45:15,214 INFO [RS:0;55d0183f16d2:40391 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-09T03:45:15,215 DEBUG [RS:0;55d0183f16d2:40391 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/55d0183f16d2:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:45:15,215 DEBUG [RS:0;55d0183f16d2:40391 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/55d0183f16d2:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:45:15,215 DEBUG [RS:0;55d0183f16d2:40391 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/55d0183f16d2:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:45:15,215 DEBUG [RS:0;55d0183f16d2:40391 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/55d0183f16d2:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:45:15,215 DEBUG [RS:0;55d0183f16d2:40391 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/55d0183f16d2:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:45:15,215 DEBUG [RS:0;55d0183f16d2:40391 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/55d0183f16d2:0, corePoolSize=2, maxPoolSize=2 2024-12-09T03:45:15,216 DEBUG [RS:0;55d0183f16d2:40391 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/55d0183f16d2:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:45:15,216 DEBUG [RS:0;55d0183f16d2:40391 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/55d0183f16d2:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:45:15,216 DEBUG [RS:0;55d0183f16d2:40391 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/55d0183f16d2:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:45:15,216 DEBUG [RS:0;55d0183f16d2:40391 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/55d0183f16d2:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:45:15,216 DEBUG [RS:0;55d0183f16d2:40391 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/55d0183f16d2:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:45:15,216 DEBUG [RS:0;55d0183f16d2:40391 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/55d0183f16d2:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:45:15,217 DEBUG [RS:0;55d0183f16d2:40391 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/55d0183f16d2:0, corePoolSize=3, maxPoolSize=3 2024-12-09T03:45:15,217 DEBUG [RS:0;55d0183f16d2:40391 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/55d0183f16d2:0, corePoolSize=3, maxPoolSize=3 2024-12-09T03:45:15,220 INFO [RS:0;55d0183f16d2:40391 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T03:45:15,220 INFO [RS:0;55d0183f16d2:40391 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T03:45:15,220 INFO [RS:0;55d0183f16d2:40391 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T03:45:15,220 INFO [RS:0;55d0183f16d2:40391 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-09T03:45:15,221 INFO [RS:0;55d0183f16d2:40391 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-09T03:45:15,221 INFO [RS:0;55d0183f16d2:40391 {}] hbase.ChoreService(168): Chore ScheduledChore name=55d0183f16d2,40391,1733715913981-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T03:45:15,242 INFO [RS:0;55d0183f16d2:40391 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-09T03:45:15,244 INFO [RS:0;55d0183f16d2:40391 {}] hbase.ChoreService(168): Chore ScheduledChore name=55d0183f16d2,40391,1733715913981-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T03:45:15,244 INFO [RS:0;55d0183f16d2:40391 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T03:45:15,245 INFO [RS:0;55d0183f16d2:40391 {}] regionserver.Replication(171): 55d0183f16d2,40391,1733715913981 started 2024-12-09T03:45:15,265 INFO [RS:0;55d0183f16d2:40391 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T03:45:15,265 INFO [RS:0;55d0183f16d2:40391 {}] regionserver.HRegionServer(1482): Serving as 55d0183f16d2,40391,1733715913981, RpcServer on 55d0183f16d2/172.17.0.2:40391, sessionid=0x1019553535e0001 2024-12-09T03:45:15,266 DEBUG [RS:0;55d0183f16d2:40391 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-09T03:45:15,266 DEBUG [RS:0;55d0183f16d2:40391 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 55d0183f16d2,40391,1733715913981 2024-12-09T03:45:15,267 DEBUG [RS:0;55d0183f16d2:40391 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '55d0183f16d2,40391,1733715913981' 2024-12-09T03:45:15,267 DEBUG [RS:0;55d0183f16d2:40391 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-09T03:45:15,268 DEBUG [RS:0;55d0183f16d2:40391 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-09T03:45:15,269 DEBUG [RS:0;55d0183f16d2:40391 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-09T03:45:15,269 DEBUG [RS:0;55d0183f16d2:40391 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-09T03:45:15,269 DEBUG [RS:0;55d0183f16d2:40391 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 55d0183f16d2,40391,1733715913981 2024-12-09T03:45:15,269 DEBUG [RS:0;55d0183f16d2:40391 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '55d0183f16d2,40391,1733715913981' 2024-12-09T03:45:15,269 DEBUG [RS:0;55d0183f16d2:40391 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-09T03:45:15,270 DEBUG [RS:0;55d0183f16d2:40391 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-09T03:45:15,271 DEBUG [RS:0;55d0183f16d2:40391 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-09T03:45:15,271 INFO [RS:0;55d0183f16d2:40391 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-09T03:45:15,271 INFO [RS:0;55d0183f16d2:40391 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-09T03:45:15,321 WARN [55d0183f16d2:39199 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T03:45:15,377 INFO [RS:0;55d0183f16d2:40391 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T03:45:15,378 INFO [RS:0;55d0183f16d2:40391 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=55d0183f16d2%2C40391%2C1733715913981, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/WALs/55d0183f16d2,40391,1733715913981, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/oldWALs, maxLogs=55 2024-12-09T03:45:15,401 INFO [RS:0;55d0183f16d2:40391 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/WALs/55d0183f16d2,40391,1733715913981/55d0183f16d2%2C40391%2C1733715913981.1733715915381 2024-12-09T03:45:15,402 DEBUG [RS:0;55d0183f16d2:40391 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-09T03:45:15,574 DEBUG [55d0183f16d2:39199 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-09T03:45:15,590 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=55d0183f16d2,40391,1733715913981 2024-12-09T03:45:15,596 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 55d0183f16d2,40391,1733715913981, state=OPENING 2024-12-09T03:45:15,602 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-09T03:45:15,603 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39199-0x1019553535e0000, quorum=127.0.0.1:60363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:45:15,603 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40391-0x1019553535e0001, quorum=127.0.0.1:60363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:45:15,604 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T03:45:15,604 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T03:45:15,606 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T03:45:15,608 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=55d0183f16d2,40391,1733715913981}] 2024-12-09T03:45:15,780 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T03:45:15,784 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33207, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T03:45:15,791 INFO [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-09T03:45:15,791 INFO [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T03:45:15,792 INFO [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-09T03:45:15,793 INFO [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=55d0183f16d2%2C40391%2C1733715913981.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/WALs/55d0183f16d2,40391,1733715913981, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/oldWALs, maxLogs=55 2024-12-09T03:45:15,814 INFO [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/WALs/55d0183f16d2,40391,1733715913981/55d0183f16d2%2C40391%2C1733715913981.meta.1733715915795.meta 2024-12-09T03:45:15,814 DEBUG [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-09T03:45:15,815 DEBUG [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-09T03:45:15,817 DEBUG [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-09T03:45:15,820 DEBUG [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-09T03:45:15,825 INFO [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-09T03:45:15,830 DEBUG [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-09T03:45:15,831 DEBUG [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T03:45:15,831 DEBUG [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-09T03:45:15,831 DEBUG [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-09T03:45:15,833 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-09T03:45:15,834 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T03:45:15,834 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:45:15,834 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T03:45:15,835 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-09T03:45:15,835 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-09T03:45:15,836 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:45:15,836 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T03:45:15,836 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-09T03:45:15,837 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T03:45:15,837 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:45:15,837 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T03:45:15,837 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-09T03:45:15,838 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T03:45:15,838 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:45:15,838 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T03:45:15,839 DEBUG [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-09T03:45:15,840 DEBUG [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/data/hbase/meta/1588230740 2024-12-09T03:45:15,841 DEBUG [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/data/hbase/meta/1588230740 2024-12-09T03:45:15,841 DEBUG [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-09T03:45:15,841 DEBUG [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-09T03:45:15,842 DEBUG [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T03:45:15,843 DEBUG [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-09T03:45:15,844 INFO [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69941582, jitterRate=0.042210787534713745}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T03:45:15,844 DEBUG [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-09T03:45:15,845 DEBUG [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733715915832Writing region info on filesystem at 1733715915832Initializing all the Stores at 1733715915833 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733715915833Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733715915833Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733715915833Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733715915833Cleaning up temporary data from old regions at 1733715915841 (+8 ms)Running coprocessor post-open hooks at 1733715915844 (+3 ms)Region opened successfully at 1733715915845 (+1 ms) 2024-12-09T03:45:15,851 INFO [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733715915771 2024-12-09T03:45:15,862 DEBUG [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-09T03:45:15,863 INFO [RS_OPEN_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-09T03:45:15,863 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=55d0183f16d2,40391,1733715913981 2024-12-09T03:45:15,864 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 55d0183f16d2,40391,1733715913981, state=OPEN 2024-12-09T03:45:15,869 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40391-0x1019553535e0001, quorum=127.0.0.1:60363, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T03:45:15,869 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39199-0x1019553535e0000, quorum=127.0.0.1:60363, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T03:45:15,869 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T03:45:15,869 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T03:45:15,870 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=55d0183f16d2,40391,1733715913981 2024-12-09T03:45:15,873 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-09T03:45:15,874 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=55d0183f16d2,40391,1733715913981 in 263 msec 2024-12-09T03:45:15,877 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-09T03:45:15,877 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 716 msec 2024-12-09T03:45:15,878 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T03:45:15,878 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-09T03:45:15,899 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T03:45:15,901 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=55d0183f16d2,40391,1733715913981, seqNum=-1] 2024-12-09T03:45:15,921 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T03:45:15,923 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43957, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T03:45:15,940 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 972 msec 2024-12-09T03:45:15,940 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733715915940, completionTime=-1 2024-12-09T03:45:15,943 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-09T03:45:15,943 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-09T03:45:15,972 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-09T03:45:15,973 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733715975973 2024-12-09T03:45:15,973 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733716035973 2024-12-09T03:45:15,973 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 29 msec 2024-12-09T03:45:15,976 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=55d0183f16d2,39199,1733715913214-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T03:45:15,976 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=55d0183f16d2,39199,1733715913214-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T03:45:15,976 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=55d0183f16d2,39199,1733715913214-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T03:45:15,977 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-55d0183f16d2:39199, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T03:45:15,978 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-09T03:45:15,978 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-09T03:45:15,984 DEBUG [master/55d0183f16d2:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-09T03:45:16,004 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.826sec 2024-12-09T03:45:16,005 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-09T03:45:16,007 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-09T03:45:16,008 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-09T03:45:16,008 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-09T03:45:16,008 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-09T03:45:16,009 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=55d0183f16d2,39199,1733715913214-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T03:45:16,010 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=55d0183f16d2,39199,1733715913214-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-09T03:45:16,014 DEBUG [master/55d0183f16d2:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-09T03:45:16,015 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8ecde90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:45:16,015 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-09T03:45:16,015 INFO [master/55d0183f16d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=55d0183f16d2,39199,1733715913214-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T03:45:16,017 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-09T03:45:16,017 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-09T03:45:16,021 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 55d0183f16d2,39199,-1 for getting cluster id 2024-12-09T03:45:16,024 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T03:45:16,033 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '167d8e73-c81d-4579-b088-40d1926afeae' 2024-12-09T03:45:16,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T03:45:16,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "167d8e73-c81d-4579-b088-40d1926afeae" 2024-12-09T03:45:16,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@690a4e78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:45:16,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [55d0183f16d2,39199,-1] 2024-12-09T03:45:16,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T03:45:16,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:45:16,057 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40638, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T03:45:16,061 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61942613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:45:16,062 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T03:45:16,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=55d0183f16d2,40391,1733715913981, seqNum=-1] 2024-12-09T03:45:16,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T03:45:16,076 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42984, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T03:45:16,095 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=55d0183f16d2,39199,1733715913214 2024-12-09T03:45:16,099 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T03:45:16,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 55d0183f16d2,39199,1733715913214 2024-12-09T03:45:16,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@930a712 2024-12-09T03:45:16,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T03:45:16,109 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40640, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T03:45:16,114 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39199 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'table1', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T03:45:16,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39199 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=table1 2024-12-09T03:45:16,123 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T03:45:16,126 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39199 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "table1" procId is: 4 2024-12-09T03:45:16,126 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:45:16,128 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T03:45:16,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39199 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T03:45:16,143 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 550c26a666e9358e823e1574d42b286d, NAME => 'table1,,1733715916110.550c26a666e9358e823e1574d42b286d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b 2024-12-09T03:45:16,150 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(898): Instantiated table1,,1733715916110.550c26a666e9358e823e1574d42b286d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T03:45:16,151 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1722): Closing 550c26a666e9358e823e1574d42b286d, disabling compactions & flushes 2024-12-09T03:45:16,151 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1755): Closing region table1,,1733715916110.550c26a666e9358e823e1574d42b286d. 2024-12-09T03:45:16,151 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1733715916110.550c26a666e9358e823e1574d42b286d. 2024-12-09T03:45:16,151 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on table1,,1733715916110.550c26a666e9358e823e1574d42b286d. after waiting 0 ms 2024-12-09T03:45:16,151 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region table1,,1733715916110.550c26a666e9358e823e1574d42b286d. 2024-12-09T03:45:16,151 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1973): Closed table1,,1733715916110.550c26a666e9358e823e1574d42b286d. 2024-12-09T03:45:16,151 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1676): Region close journal for 550c26a666e9358e823e1574d42b286d: Waiting for close lock at 1733715916151Disabling compacts and flushes for region at 1733715916151Disabling writes for close at 1733715916151Writing region close event to WAL at 1733715916151Closed at 1733715916151 2024-12-09T03:45:16,153 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T03:45:16,160 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"table1,,1733715916110.550c26a666e9358e823e1574d42b286d.","families":{"info":[{"qualifier":"regioninfo","vlen":40,"tag":[],"timestamp":"1733715916153"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733715916153"}]},"ts":"1733715916153"} 2024-12-09T03:45:16,165 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T03:45:16,166 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T03:45:16,169 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733715916166"}]},"ts":"1733715916166"} 2024-12-09T03:45:16,172 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=ENABLING in hbase:meta 2024-12-09T03:45:16,174 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=550c26a666e9358e823e1574d42b286d, ASSIGN}] 2024-12-09T03:45:16,176 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=550c26a666e9358e823e1574d42b286d, ASSIGN 2024-12-09T03:45:16,177 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=table1, region=550c26a666e9358e823e1574d42b286d, ASSIGN; state=OFFLINE, location=55d0183f16d2,40391,1733715913981; forceNewPlan=false, retain=false 2024-12-09T03:45:16,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39199 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T03:45:16,329 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=550c26a666e9358e823e1574d42b286d, regionState=OPENING, regionLocation=55d0183f16d2,40391,1733715913981 2024-12-09T03:45:16,332 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=table1, region=550c26a666e9358e823e1574d42b286d, ASSIGN because future has completed 2024-12-09T03:45:16,333 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 550c26a666e9358e823e1574d42b286d, server=55d0183f16d2,40391,1733715913981}] 2024-12-09T03:45:16,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39199 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T03:45:16,489 INFO [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open table1,,1733715916110.550c26a666e9358e823e1574d42b286d. 2024-12-09T03:45:16,490 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 550c26a666e9358e823e1574d42b286d, NAME => 'table1,,1733715916110.550c26a666e9358e823e1574d42b286d.', STARTKEY => '', ENDKEY => ''} 2024-12-09T03:45:16,490 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 550c26a666e9358e823e1574d42b286d 2024-12-09T03:45:16,490 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated table1,,1733715916110.550c26a666e9358e823e1574d42b286d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T03:45:16,491 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 550c26a666e9358e823e1574d42b286d 2024-12-09T03:45:16,491 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 550c26a666e9358e823e1574d42b286d 2024-12-09T03:45:16,492 INFO [StoreOpener-550c26a666e9358e823e1574d42b286d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 550c26a666e9358e823e1574d42b286d 2024-12-09T03:45:16,496 INFO [StoreOpener-550c26a666e9358e823e1574d42b286d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 550c26a666e9358e823e1574d42b286d columnFamilyName family 2024-12-09T03:45:16,496 DEBUG [StoreOpener-550c26a666e9358e823e1574d42b286d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:45:16,497 INFO [StoreOpener-550c26a666e9358e823e1574d42b286d-1 {}] regionserver.HStore(327): Store=550c26a666e9358e823e1574d42b286d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:45:16,497 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 550c26a666e9358e823e1574d42b286d 2024-12-09T03:45:16,498 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/data/default/table1/550c26a666e9358e823e1574d42b286d 2024-12-09T03:45:16,498 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/data/default/table1/550c26a666e9358e823e1574d42b286d 2024-12-09T03:45:16,498 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 550c26a666e9358e823e1574d42b286d 2024-12-09T03:45:16,498 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 550c26a666e9358e823e1574d42b286d 2024-12-09T03:45:16,499 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 550c26a666e9358e823e1574d42b286d 2024-12-09T03:45:16,505 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/data/default/table1/550c26a666e9358e823e1574d42b286d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T03:45:16,506 INFO [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 550c26a666e9358e823e1574d42b286d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72127247, jitterRate=0.07477973401546478}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T03:45:16,506 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 550c26a666e9358e823e1574d42b286d 2024-12-09T03:45:16,507 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 550c26a666e9358e823e1574d42b286d: Running coprocessor pre-open hook at 1733715916491Writing region info on filesystem at 1733715916491Initializing all the Stores at 1733715916491Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733715916491Cleaning up temporary data from old regions at 1733715916498 (+7 ms)Running coprocessor post-open hooks at 1733715916506 (+8 ms)Region opened successfully at 1733715916507 (+1 ms) 2024-12-09T03:45:16,508 INFO [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for table1,,1733715916110.550c26a666e9358e823e1574d42b286d., pid=6, masterSystemTime=1733715916486 2024-12-09T03:45:16,511 DEBUG [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for table1,,1733715916110.550c26a666e9358e823e1574d42b286d. 2024-12-09T03:45:16,511 INFO [RS_OPEN_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened table1,,1733715916110.550c26a666e9358e823e1574d42b286d. 2024-12-09T03:45:16,512 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=550c26a666e9358e823e1574d42b286d, regionState=OPEN, openSeqNum=2, regionLocation=55d0183f16d2,40391,1733715913981 2024-12-09T03:45:16,515 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 550c26a666e9358e823e1574d42b286d, server=55d0183f16d2,40391,1733715913981 because future has completed 2024-12-09T03:45:16,519 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-09T03:45:16,519 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 550c26a666e9358e823e1574d42b286d, server=55d0183f16d2,40391,1733715913981 in 184 msec 2024-12-09T03:45:16,521 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-09T03:45:16,521 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=550c26a666e9358e823e1574d42b286d, ASSIGN in 345 msec 2024-12-09T03:45:16,521 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T03:45:16,522 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733715916522"}]},"ts":"1733715916522"} 2024-12-09T03:45:16,525 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=ENABLED in hbase:meta 2024-12-09T03:45:16,526 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T03:45:16,527 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=table1 in 409 msec 2024-12-09T03:45:16,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39199 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T03:45:16,759 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:table1 completed 2024-12-09T03:45:16,759 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table table1 get assigned. Timeout = 60000ms 2024-12-09T03:45:16,762 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T03:45:16,768 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table table1 assigned to meta. Checking AM states. 2024-12-09T03:45:16,768 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T03:45:16,769 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table table1 assigned. 2024-12-09T03:45:16,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row1', locateType=CURRENT is [region=table1,,1733715916110.550c26a666e9358e823e1574d42b286d., hostname=55d0183f16d2,40391,1733715913981, seqNum=2] 2024-12-09T03:45:16,788 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 550c26a666e9358e823e1574d42b286d 1/1 column families, dataSize=78 B heapSize=480 B 2024-12-09T03:45:16,814 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 403df871ec8e4f85befacff9fd9a62c9 2024-12-09T03:45:16,853 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/data/default/table1/550c26a666e9358e823e1574d42b286d/.tmp/family/403df871ec8e4f85befacff9fd9a62c9 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-12-09T03:45:16,858 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/data/default/table1/550c26a666e9358e823e1574d42b286d/.tmp/family/403df871ec8e4f85befacff9fd9a62c9 2024-12-09T03:45:16,894 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/data/default/table1/550c26a666e9358e823e1574d42b286d/.tmp/family/403df871ec8e4f85befacff9fd9a62c9 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/data/default/table1/550c26a666e9358e823e1574d42b286d/family/403df871ec8e4f85befacff9fd9a62c9 2024-12-09T03:45:16,895 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/data/default/table1/550c26a666e9358e823e1574d42b286d/family/403df871ec8e4f85befacff9fd9a62c9, delay=951 ms 2024-12-09T03:45:16,897 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/data/default/table1/550c26a666e9358e823e1574d42b286d/family/403df871ec8e4f85befacff9fd9a62c9, entries=2, sequenceid=6, filesize=4.9 K 2024-12-09T03:45:16,903 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 550c26a666e9358e823e1574d42b286d in 114ms, sequenceid=6, compaction requested=false 2024-12-09T03:45:16,904 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'table1' 2024-12-09T03:45:16,905 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 550c26a666e9358e823e1574d42b286d: 2024-12-09T03:45:17,849 DEBUG [hfile-prefetch-1733715916895 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 403df871ec8e4f85befacff9fd9a62c9_0. 2024-12-09T03:45:17,853 DEBUG [hfile-prefetch-1733715916895 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 403df871ec8e4f85befacff9fd9a62c9_117. 2024-12-09T03:45:17,855 DEBUG [hfile-prefetch-1733715916895 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/data/default/table1/550c26a666e9358e823e1574d42b286d/family/403df871ec8e4f85befacff9fd9a62c9, with total blocks 2, and data blocks 1 2024-12-09T03:45:17,855 DEBUG [hfile-prefetch-1733715916895 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-09T03:45:17,855 DEBUG [hfile-prefetch-1733715916895 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/data/default/table1/550c26a666e9358e823e1574d42b286d/family/403df871ec8e4f85befacff9fd9a62c9, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-12-09T03:45:17,856 DEBUG [hfile-prefetch-1733715916895 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/data/default/table1/550c26a666e9358e823e1574d42b286d/family/403df871ec8e4f85befacff9fd9a62c9. Will sleep 100ms and try the verification again. 2024-12-09T03:45:17,956 DEBUG [hfile-prefetch-1733715916895 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/data/default/table1/550c26a666e9358e823e1574d42b286d/family/403df871ec8e4f85befacff9fd9a62c9, with total blocks 2, and data blocks 1 2024-12-09T03:45:17,956 DEBUG [hfile-prefetch-1733715916895 {}] bucket.BucketCache(2285): iterating over 1 entries in the backing map 2024-12-09T03:45:17,956 DEBUG [hfile-prefetch-1733715916895 {}] bucket.BucketCache(2293): found block for file 403df871ec8e4f85befacff9fd9a62c9 in the backing map. Acquiring read lock for offset 0 2024-12-09T03:45:17,956 DEBUG [hfile-prefetch-1733715916895 {}] bucket.BucketCache(2305): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/data/default/table1/550c26a666e9358e823e1574d42b286d/family/403df871ec8e4f85befacff9fd9a62c9 has now been fully cached. 2024-12-09T03:45:17,957 DEBUG [hfile-prefetch-1733715916895 {}] hfile.PrefetchExecutor(121): Prefetch completed for 403df871ec8e4f85befacff9fd9a62c9 2024-12-09T03:45:18,066 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-09T03:45:18,066 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-09T03:45:18,067 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-09T03:45:18,067 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-12-09T03:45:18,089 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-12-09T03:45:18,089 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1733715918066 2024-12-09T03:45:18,090 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-09T03:45:18,909 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T03:45:18,912 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42994, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T03:45:18,915 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;55d0183f16d2:40391,5,FailOnTimeoutGroup] 2024-12-09T03:45:18,915 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '55d0183f16d2,40391,1733715913981' ***** 2024-12-09T03:45:18,915 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-12-09T03:45:18,915 INFO [Time-limited test {}] hfile.TestPrefetchRSClose(129): Stopped Region Server 0. 2024-12-09T03:45:18,916 INFO [RS:0;55d0183f16d2:40391 {}] bucket.BucketCache(1738): Disabling cache 2024-12-09T03:45:18,916 INFO [RS:0;55d0183f16d2:40391 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/bucket.persistence 2024-12-09T03:45:18,916 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-09T03:45:18,916 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-09T03:45:18,916 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-09T03:45:18,916 INFO [RS:0;55d0183f16d2:40391 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-09T03:45:18,916 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-09T03:45:18,922 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-09T03:45:18,924 INFO [regionserver/55d0183f16d2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T03:45:18,927 DEBUG [RS:0;55d0183f16d2:40391 {}] bucket.BucketCache(1487): Thread RS:0;55d0183f16d2:40391 started persisting bucket cache to file 2024-12-09T03:45:18,927 DEBUG [RS:0;55d0183f16d2:40391 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-09T03:45:18,927 DEBUG [RS:0;55d0183f16d2:40391 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-12-09T03:45:18,930 DEBUG [RS:0;55d0183f16d2:40391 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-12-09T03:45:18,930 DEBUG [RS:0;55d0183f16d2:40391 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1733715918927 2024-12-09T03:45:18,930 DEBUG [RS:0;55d0183f16d2:40391 {}] bucket.BucketCache(1509): Thread RS:0;55d0183f16d2:40391 finished persisting bucket cache to file, renaming 2024-12-09T03:45:18,931 INFO [RS:0;55d0183f16d2:40391 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-09T03:45:18,931 INFO [RS:0;55d0183f16d2:40391 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-09T03:45:18,931 INFO [RS:0;55d0183f16d2:40391 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-09T03:45:18,931 INFO [RS:0;55d0183f16d2:40391 {}] regionserver.HRegionServer(3091): Received CLOSE for 550c26a666e9358e823e1574d42b286d 2024-12-09T03:45:18,932 INFO [RS:0;55d0183f16d2:40391 {}] regionserver.HRegionServer(959): stopping server 55d0183f16d2,40391,1733715913981 2024-12-09T03:45:18,932 INFO [RS:0;55d0183f16d2:40391 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T03:45:18,932 INFO [RS:0;55d0183f16d2:40391 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;55d0183f16d2:40391. 2024-12-09T03:45:18,932 INFO [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region table1,,1733715916110.550c26a666e9358e823e1574d42b286d. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-09T03:45:18,933 DEBUG [RS:0;55d0183f16d2:40391 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T03:45:18,933 DEBUG [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 550c26a666e9358e823e1574d42b286d, disabling compactions & flushes 2024-12-09T03:45:18,933 INFO [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region table1,,1733715916110.550c26a666e9358e823e1574d42b286d. 2024-12-09T03:45:18,933 DEBUG [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1733715916110.550c26a666e9358e823e1574d42b286d. 2024-12-09T03:45:18,933 DEBUG [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on table1,,1733715916110.550c26a666e9358e823e1574d42b286d. after waiting 0 ms 2024-12-09T03:45:18,933 DEBUG [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region table1,,1733715916110.550c26a666e9358e823e1574d42b286d. 2024-12-09T03:45:18,938 DEBUG [RS:0;55d0183f16d2:40391 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:45:18,938 INFO [RS:0;55d0183f16d2:40391 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-09T03:45:18,938 INFO [RS:0;55d0183f16d2:40391 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-09T03:45:18,938 INFO [RS:0;55d0183f16d2:40391 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-09T03:45:18,938 INFO [RS:0;55d0183f16d2:40391 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-09T03:45:18,939 INFO [RS:0;55d0183f16d2:40391 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-09T03:45:18,939 DEBUG [RS:0;55d0183f16d2:40391 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 550c26a666e9358e823e1574d42b286d=table1,,1733715916110.550c26a666e9358e823e1574d42b286d.} 2024-12-09T03:45:18,939 INFO [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-09T03:45:18,939 DEBUG [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-09T03:45:18,939 DEBUG [RS:0;55d0183f16d2:40391 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 550c26a666e9358e823e1574d42b286d 2024-12-09T03:45:18,939 INFO [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-09T03:45:18,939 DEBUG [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-09T03:45:18,939 DEBUG [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T03:45:18,940 DEBUG [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T03:45:18,940 INFO [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.20 KB heapSize=3.23 KB 2024-12-09T03:45:18,940 DEBUG [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/data/default/table1/550c26a666e9358e823e1574d42b286d/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-09T03:45:18,942 INFO [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed table1,,1733715916110.550c26a666e9358e823e1574d42b286d. 2024-12-09T03:45:18,942 DEBUG [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 550c26a666e9358e823e1574d42b286d: Waiting for close lock at 1733715918932Running coprocessor pre-close hooks at 1733715918933 (+1 ms)Disabling compacts and flushes for region at 1733715918933Disabling writes for close at 1733715918933Writing region close event to WAL at 1733715918935 (+2 ms)Running coprocessor post-close hooks at 1733715918941 (+6 ms)Closed at 1733715918942 (+1 ms) 2024-12-09T03:45:18,942 DEBUG [RS_CLOSE_REGION-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed table1,,1733715916110.550c26a666e9358e823e1574d42b286d. 2024-12-09T03:45:18,953 DEBUG [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 072dedf734584e6c865c8bccfce7efe9 2024-12-09T03:45:18,973 DEBUG [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/data/hbase/meta/1588230740/.tmp/info/072dedf734584e6c865c8bccfce7efe9 is 133, key is table1,,1733715916110.550c26a666e9358e823e1574d42b286d./info:regioninfo/1733715916512/Put/seqid=0 2024-12-09T03:45:18,975 INFO [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/data/hbase/meta/1588230740/.tmp/info/072dedf734584e6c865c8bccfce7efe9 2024-12-09T03:45:18,986 DEBUG [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 49280634f4f94df290e278b799e89b7e 2024-12-09T03:45:19,002 DEBUG [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/data/hbase/meta/1588230740/.tmp/ns/49280634f4f94df290e278b799e89b7e is 43, key is default/ns:d/1733715915927/Put/seqid=0 2024-12-09T03:45:19,003 INFO [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/data/hbase/meta/1588230740/.tmp/ns/49280634f4f94df290e278b799e89b7e 2024-12-09T03:45:19,014 DEBUG [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on e15dc34840c54babbd068b93615922fa 2024-12-09T03:45:19,028 DEBUG [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/data/hbase/meta/1588230740/.tmp/table/e15dc34840c54babbd068b93615922fa is 42, key is table1/table:state/1733715916522/Put/seqid=0 2024-12-09T03:45:19,029 INFO [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=76 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/data/hbase/meta/1588230740/.tmp/table/e15dc34840c54babbd068b93615922fa 2024-12-09T03:45:19,032 DEBUG [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/data/hbase/meta/1588230740/.tmp/info/072dedf734584e6c865c8bccfce7efe9 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/data/hbase/meta/1588230740/info/072dedf734584e6c865c8bccfce7efe9 2024-12-09T03:45:19,033 DEBUG [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/data/hbase/meta/1588230740/info/072dedf734584e6c865c8bccfce7efe9, delay=976 ms 2024-12-09T03:45:19,034 INFO [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/data/hbase/meta/1588230740/info/072dedf734584e6c865c8bccfce7efe9, entries=10, sequenceid=11, filesize=6.3 K 2024-12-09T03:45:19,035 DEBUG [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/data/hbase/meta/1588230740/.tmp/ns/49280634f4f94df290e278b799e89b7e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/data/hbase/meta/1588230740/ns/49280634f4f94df290e278b799e89b7e 2024-12-09T03:45:19,036 DEBUG [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/data/hbase/meta/1588230740/ns/49280634f4f94df290e278b799e89b7e, delay=900 ms 2024-12-09T03:45:19,038 INFO [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/data/hbase/meta/1588230740/ns/49280634f4f94df290e278b799e89b7e, entries=2, sequenceid=11, filesize=5.0 K 2024-12-09T03:45:19,038 DEBUG [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/data/hbase/meta/1588230740/.tmp/table/e15dc34840c54babbd068b93615922fa as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/data/hbase/meta/1588230740/table/e15dc34840c54babbd068b93615922fa 2024-12-09T03:45:19,039 DEBUG [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/data/hbase/meta/1588230740/table/e15dc34840c54babbd068b93615922fa, delay=997 ms 2024-12-09T03:45:19,041 INFO [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/data/hbase/meta/1588230740/table/e15dc34840c54babbd068b93615922fa, entries=2, sequenceid=11, filesize=5.1 K 2024-12-09T03:45:19,042 INFO [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.20 KB/1227, heapSize ~2.94 KB/3008, currentSize=0 B/0 for 1588230740 in 101ms, sequenceid=11, compaction requested=false 2024-12-09T03:45:19,042 DEBUG [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-09T03:45:19,043 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/data/hbase/meta/1588230740/info/072dedf734584e6c865c8bccfce7efe9 2024-12-09T03:45:19,044 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/data/hbase/meta/1588230740/ns/49280634f4f94df290e278b799e89b7e 2024-12-09T03:45:19,045 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/data/hbase/meta/1588230740/table/e15dc34840c54babbd068b93615922fa 2024-12-09T03:45:19,049 DEBUG [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-12-09T03:45:19,050 DEBUG [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-09T03:45:19,050 INFO [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-09T03:45:19,050 DEBUG [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733715918939Running coprocessor pre-close hooks at 1733715918939Disabling compacts and flushes for region at 1733715918939Disabling writes for close at 1733715918939Obtaining lock to block concurrent updates at 1733715918940 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1733715918940Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1227, getHeapSize=3248, getOffHeapSize=0, getCellsCount=14 at 1733715918942 (+2 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733715918942Flushing 1588230740/info: creating writer at 1733715918943 (+1 ms)Flushing 1588230740/info: appending metadata at 1733715918971 (+28 ms)Flushing 1588230740/info: closing flushed file at 1733715918972 (+1 ms)Flushing 1588230740/ns: creating writer at 1733715918979 (+7 ms)Flushing 1588230740/ns: appending metadata at 1733715919001 (+22 ms)Flushing 1588230740/ns: closing flushed file at 1733715919001Flushing 1588230740/table: creating writer at 1733715919007 (+6 ms)Flushing 1588230740/table: appending metadata at 1733715919028 (+21 ms)Flushing 1588230740/table: closing flushed file at 1733715919028Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@43cf04bd: reopening flushed file at 1733715919031 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@63339184: reopening flushed file at 1733715919035 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@12f2d5bb: reopening flushed file at 1733715919038 (+3 ms)Finished flush of dataSize ~1.20 KB/1227, heapSize ~2.94 KB/3008, currentSize=0 B/0 for 1588230740 in 101ms, sequenceid=11, compaction requested=false at 1733715919042 (+4 ms)Writing region close event to WAL at 1733715919045 (+3 ms)Running coprocessor post-close hooks at 1733715919050 (+5 ms)Closed at 1733715919050 2024-12-09T03:45:19,050 DEBUG [RS_CLOSE_META-regionserver/55d0183f16d2:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-09T03:45:19,140 INFO [RS:0;55d0183f16d2:40391 {}] regionserver.HRegionServer(976): stopping server 55d0183f16d2,40391,1733715913981; all regions closed. 2024-12-09T03:45:19,144 DEBUG [RS:0;55d0183f16d2:40391 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/oldWALs 2024-12-09T03:45:19,145 INFO [RS:0;55d0183f16d2:40391 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 55d0183f16d2%2C40391%2C1733715913981.meta:.meta(num 1733715915795) 2024-12-09T03:45:19,146 DEBUG [RS:0;55d0183f16d2:40391 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/oldWALs 2024-12-09T03:45:19,147 INFO [RS:0;55d0183f16d2:40391 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 55d0183f16d2%2C40391%2C1733715913981:(num 1733715915381) 2024-12-09T03:45:19,147 DEBUG [RS:0;55d0183f16d2:40391 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:45:19,147 INFO [RS:0;55d0183f16d2:40391 {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T03:45:19,147 INFO [RS:0;55d0183f16d2:40391 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T03:45:19,147 INFO [RS:0;55d0183f16d2:40391 {}] hbase.ChoreService(370): Chore service for: regionserver/55d0183f16d2:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-09T03:45:19,147 INFO [RS:0;55d0183f16d2:40391 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T03:45:19,147 INFO [regionserver/55d0183f16d2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T03:45:19,148 INFO [RS:0;55d0183f16d2:40391 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40391 2024-12-09T03:45:19,156 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40391-0x1019553535e0001, quorum=127.0.0.1:60363, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/55d0183f16d2,40391,1733715913981 2024-12-09T03:45:19,156 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39199-0x1019553535e0000, quorum=127.0.0.1:60363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T03:45:19,156 INFO [RS:0;55d0183f16d2:40391 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T03:45:19,158 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [55d0183f16d2,40391,1733715913981] 2024-12-09T03:45:19,160 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/55d0183f16d2,40391,1733715913981 already deleted, retry=false 2024-12-09T03:45:19,160 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 55d0183f16d2,40391,1733715913981 on 55d0183f16d2,39199,1733715913214 2024-12-09T03:45:19,165 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 55d0183f16d2,40391,1733715913981, splitWal=true, meta=true 2024-12-09T03:45:19,167 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 55d0183f16d2,40391,1733715913981, splitWal=true, meta=true 2024-12-09T03:45:19,167 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 55d0183f16d2,40391,1733715913981 (carryingMeta=true) 55d0183f16d2,40391,1733715913981/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@a10b6b4[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-09T03:45:19,168 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 55d0183f16d2,40391,1733715913981, splitWal=true, meta=true, isMeta: true 2024-12-09T03:45:19,169 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/WALs/55d0183f16d2,40391,1733715913981-splitting 2024-12-09T03:45:19,169 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/WALs/55d0183f16d2,40391,1733715913981-splitting dir is empty, no logs to split. 2024-12-09T03:45:19,169 INFO [PEWorker-2 {}] master.SplitWALManager(105): 55d0183f16d2,40391,1733715913981 WAL count=0, meta=true 2024-12-09T03:45:19,171 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/WALs/55d0183f16d2,40391,1733715913981-splitting dir is empty, no logs to split. 2024-12-09T03:45:19,171 INFO [PEWorker-2 {}] master.SplitWALManager(105): 55d0183f16d2,40391,1733715913981 WAL count=0, meta=true 2024-12-09T03:45:19,171 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 55d0183f16d2,40391,1733715913981 WAL splitting is done? wals=0, meta=true 2024-12-09T03:45:19,172 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-09T03:45:19,173 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T03:45:19,174 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-09T03:45:19,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40391-0x1019553535e0001, quorum=127.0.0.1:60363, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:45:19,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40391-0x1019553535e0001, quorum=127.0.0.1:60363, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:45:19,259 INFO [RS:0;55d0183f16d2:40391 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T03:45:19,260 INFO [RS:0;55d0183f16d2:40391 {}] regionserver.HRegionServer(1031): Exiting; stopping=55d0183f16d2,40391,1733715913981; zookeeper connection closed. 2024-12-09T03:45:19,260 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7488bee {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7488bee 2024-12-09T03:45:19,325 WARN [55d0183f16d2:39199 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T03:45:19,916 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-09T03:45:19,916 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T03:45:19,916 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestPrefetchRSClose.tearDown(TestPrefetchRSClose.java:136) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T03:45:19,916 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:45:19,917 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:45:19,917 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T03:45:19,917 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-09T03:45:19,917 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=365092990, stopped=false 2024-12-09T03:45:19,918 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=55d0183f16d2,39199,1733715913214 2024-12-09T03:45:19,918 INFO [Time-limited test {}] master.HMaster(3321): ***** STOPPING master '55d0183f16d2,39199,1733715913214' ***** 2024-12-09T03:45:19,918 INFO [Time-limited test {}] master.HMaster(3323): STOPPED: OnlineServer=0 right after cluster shutdown set 2024-12-09T03:45:19,918 INFO [M:0;55d0183f16d2:39199 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T03:45:19,918 INFO [M:0;55d0183f16d2:39199 {}] client.AsyncConnectionImpl(233): Connection has been closed by M:0;55d0183f16d2:39199. 2024-12-09T03:45:19,918 DEBUG [M:0;55d0183f16d2:39199 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:630) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T03:45:19,918 DEBUG [M:0;55d0183f16d2:39199 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:45:19,918 INFO [M:0;55d0183f16d2:39199 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T03:45:19,919 DEBUG [M:0;55d0183f16d2:39199 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-09T03:45:19,919 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-09T03:45:19,919 DEBUG [M:0;55d0183f16d2:39199 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-09T03:45:19,919 DEBUG [master/55d0183f16d2:0:becomeActiveMaster-HFileCleaner.large.0-1733715915078 {}] cleaner.HFileCleaner(306): Exit Thread[master/55d0183f16d2:0:becomeActiveMaster-HFileCleaner.large.0-1733715915078,5,FailOnTimeoutGroup] 2024-12-09T03:45:19,919 DEBUG [master/55d0183f16d2:0:becomeActiveMaster-HFileCleaner.small.0-1733715915079 {}] cleaner.HFileCleaner(306): Exit Thread[master/55d0183f16d2:0:becomeActiveMaster-HFileCleaner.small.0-1733715915079,5,FailOnTimeoutGroup] 2024-12-09T03:45:19,919 INFO [M:0;55d0183f16d2:39199 {}] hbase.ChoreService(370): Chore service for: master/55d0183f16d2:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-09T03:45:19,919 INFO [M:0;55d0183f16d2:39199 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T03:45:19,920 DEBUG [M:0;55d0183f16d2:39199 {}] master.HMaster(1795): Stopping service threads 2024-12-09T03:45:19,920 INFO [M:0;55d0183f16d2:39199 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-09T03:45:19,920 INFO [M:0;55d0183f16d2:39199 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T03:45:19,921 INFO [M:0;55d0183f16d2:39199 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-09T03:45:19,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39199-0x1019553535e0000, quorum=127.0.0.1:60363, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-09T03:45:19,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39199-0x1019553535e0000, quorum=127.0.0.1:60363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:45:19,921 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-09T03:45:19,922 DEBUG [M:0;55d0183f16d2:39199 {}] zookeeper.ZKUtil(347): master:39199-0x1019553535e0000, quorum=127.0.0.1:60363, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-09T03:45:19,922 WARN [M:0;55d0183f16d2:39199 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-09T03:45:19,922 INFO [M:0;55d0183f16d2:39199 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/.lastflushedseqids 2024-12-09T03:45:19,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39199-0x1019553535e0000, quorum=127.0.0.1:60363, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T03:45:19,924 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-09T03:45:19,925 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39199-0x1019553535e0000, quorum=127.0.0.1:60363, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T03:45:19,936 INFO [M:0;55d0183f16d2:39199 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-09T03:45:20,075 DEBUG [55d0183f16d2:39199 {}] assignment.AssignmentManager(2455): Stopped! Dropping assign of 1 queued regions. 2024-12-09T03:45:20,075 INFO [M:0;55d0183f16d2:39199 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-09T03:45:20,076 DEBUG [M:0;55d0183f16d2:39199 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-09T03:45:20,076 INFO [M:0;55d0183f16d2:39199 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T03:45:20,076 DEBUG [M:0;55d0183f16d2:39199 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T03:45:20,076 DEBUG [M:0;55d0183f16d2:39199 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-09T03:45:20,076 DEBUG [M:0;55d0183f16d2:39199 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T03:45:20,076 INFO [M:0;55d0183f16d2:39199 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=24.17 KB heapSize=30.80 KB 2024-12-09T03:45:20,087 DEBUG [M:0;55d0183f16d2:39199 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 54af44b57c7a4b2ba06ed4a39eb95747 2024-12-09T03:45:20,102 DEBUG [M:0;55d0183f16d2:39199 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/54af44b57c7a4b2ba06ed4a39eb95747 is 82, key is hbase:meta,,1/info:regioninfo/1733715915863/Put/seqid=0 2024-12-09T03:45:20,103 INFO [M:0;55d0183f16d2:39199 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=64 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/54af44b57c7a4b2ba06ed4a39eb95747 2024-12-09T03:45:20,112 DEBUG [M:0;55d0183f16d2:39199 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on cf28be7875094db2b7e6a31b479d72b6 2024-12-09T03:45:20,126 DEBUG [M:0;55d0183f16d2:39199 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cf28be7875094db2b7e6a31b479d72b6 is 706, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733715916527/Put/seqid=0 2024-12-09T03:45:20,127 INFO [M:0;55d0183f16d2:39199 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=23.56 KB at sequenceid=64 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cf28be7875094db2b7e6a31b479d72b6 2024-12-09T03:45:20,135 DEBUG [M:0;55d0183f16d2:39199 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 7f54d30e84944da9af18cd581bade6df 2024-12-09T03:45:20,149 DEBUG [M:0;55d0183f16d2:39199 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7f54d30e84944da9af18cd581bade6df is 58, key is 55d0183f16d2,40391,1733715913981/rs:/1733715919168/DeleteFamily/seqid=0 2024-12-09T03:45:20,149 INFO [M:0;55d0183f16d2:39199 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=119 B at sequenceid=64 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7f54d30e84944da9af18cd581bade6df 2024-12-09T03:45:20,151 INFO [M:0;55d0183f16d2:39199 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7f54d30e84944da9af18cd581bade6df 2024-12-09T03:45:20,152 DEBUG [M:0;55d0183f16d2:39199 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/54af44b57c7a4b2ba06ed4a39eb95747 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/54af44b57c7a4b2ba06ed4a39eb95747 2024-12-09T03:45:20,153 INFO [M:0;55d0183f16d2:39199 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/54af44b57c7a4b2ba06ed4a39eb95747, entries=8, sequenceid=64, filesize=5.5 K 2024-12-09T03:45:20,154 DEBUG [M:0;55d0183f16d2:39199 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cf28be7875094db2b7e6a31b479d72b6 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/cf28be7875094db2b7e6a31b479d72b6 2024-12-09T03:45:20,155 INFO [M:0;55d0183f16d2:39199 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/cf28be7875094db2b7e6a31b479d72b6, entries=8, sequenceid=64, filesize=6.5 K 2024-12-09T03:45:20,156 DEBUG [M:0;55d0183f16d2:39199 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7f54d30e84944da9af18cd581bade6df as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7f54d30e84944da9af18cd581bade6df 2024-12-09T03:45:20,157 INFO [M:0;55d0183f16d2:39199 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7f54d30e84944da9af18cd581bade6df 2024-12-09T03:45:20,157 INFO [M:0;55d0183f16d2:39199 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84ae6fbb-4a9a-2636-be1a-345388f9974c/e6676edb-9961-2369-3b32-c6090504921b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7f54d30e84944da9af18cd581bade6df, entries=1, sequenceid=64, filesize=5.2 K 2024-12-09T03:45:20,158 INFO [M:0;55d0183f16d2:39199 {}] regionserver.HRegion(3140): Finished flush of dataSize ~24.17 KB/24747, heapSize ~30.51 KB/31240, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 82ms, sequenceid=64, compaction requested=false 2024-12-09T03:45:20,160 INFO [M:0;55d0183f16d2:39199 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T03:45:20,161 DEBUG [M:0;55d0183f16d2:39199 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733715920076Disabling compacts and flushes for region at 1733715920076Disabling writes for close at 1733715920076Obtaining lock to block concurrent updates at 1733715920076Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733715920076Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=24747, getHeapSize=31480, getOffHeapSize=0, getCellsCount=76 at 1733715920077 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733715920077Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733715920077Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733715920101 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733715920102 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733715920106 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733715920125 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733715920125Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733715920128 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733715920148 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733715920148Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d26a291: reopening flushed file at 1733715920152 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37c8e7c0: reopening flushed file at 1733715920154 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@587062b8: reopening flushed file at 1733715920155 (+1 ms)Finished flush of dataSize ~24.17 KB/24747, heapSize ~30.51 KB/31240, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 82ms, sequenceid=64, compaction requested=false at 1733715920158 (+3 ms)Writing region close event to WAL at 1733715920160 (+2 ms)Closed at 1733715920160 2024-12-09T03:45:20,162 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T03:45:20,162 INFO [M:0;55d0183f16d2:39199 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-09T03:45:20,163 INFO [M:0;55d0183f16d2:39199 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39199 2024-12-09T03:45:20,163 INFO [M:0;55d0183f16d2:39199 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T03:45:20,265 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39199-0x1019553535e0000, quorum=127.0.0.1:60363, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:45:20,265 INFO [M:0;55d0183f16d2:39199 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T03:45:20,265 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39199-0x1019553535e0000, quorum=127.0.0.1:60363, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:45:20,270 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-09T03:45:20,294 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-09T03:45:20,294 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T03:45:20,302 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestPrefetchRSClose#testPrefetchPersistence Thread=45 (was 12) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/55d0183f16d2:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733715919036 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733715919039 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733715916895 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: hfile-prefetch-1733715919033 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/55d0183f16d2:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/55d0183f16d2:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) - Thread LEAK? -, OpenFileDescriptor=337 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=834 (was 907), ProcessCount=11 (was 11), AvailableMemoryMB=12413 (was 12717)