2024-12-04 10:09:08,760 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-12-04 10:09:08,776 main DEBUG Took 0.014232 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-04 10:09:08,776 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-04 10:09:08,777 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-04 10:09:08,777 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-04 10:09:08,778 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 10:09:08,784 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-04 10:09:08,795 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 10:09:08,796 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 10:09:08,797 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 10:09:08,797 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 10:09:08,798 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 10:09:08,798 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 10:09:08,799 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 10:09:08,799 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 10:09:08,799 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 10:09:08,800 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 10:09:08,800 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 10:09:08,801 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 10:09:08,801 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 10:09:08,802 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 10:09:08,802 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 10:09:08,803 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 10:09:08,803 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 10:09:08,804 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 10:09:08,804 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 10:09:08,804 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 10:09:08,805 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 10:09:08,805 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 10:09:08,806 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 10:09:08,806 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 10:09:08,806 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 10:09:08,807 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-04 10:09:08,809 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 10:09:08,810 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-04 10:09:08,812 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-04 10:09:08,812 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-04 10:09:08,814 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-04 10:09:08,814 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-04 10:09:08,823 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-04 10:09:08,826 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-04 10:09:08,827 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-04 10:09:08,828 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-04 10:09:08,828 main DEBUG createAppenders(={Console}) 2024-12-04 10:09:08,829 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-12-04 10:09:08,829 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-12-04 10:09:08,829 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-12-04 10:09:08,830 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-04 10:09:08,830 main DEBUG OutputStream closed 2024-12-04 10:09:08,831 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-04 10:09:08,831 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-04 10:09:08,831 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-12-04 10:09:08,911 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-04 10:09:08,913 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-04 10:09:08,914 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-04 10:09:08,915 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-04 10:09:08,916 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-04 10:09:08,916 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-04 10:09:08,917 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-04 10:09:08,917 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-04 10:09:08,918 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-04 10:09:08,918 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-04 10:09:08,919 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-04 10:09:08,919 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-04 10:09:08,920 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-04 10:09:08,920 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-04 10:09:08,920 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-04 10:09:08,921 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-04 10:09:08,921 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-04 10:09:08,922 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-04 10:09:08,924 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-04 10:09:08,924 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-12-04 10:09:08,924 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-04 10:09:08,925 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-12-04T10:09:08,937 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-12-04 10:09:08,939 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-04 10:09:08,940 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-04T10:09:09,146 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966 2024-12-04T10:09:09,168 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=850, ProcessCount=11, AvailableMemoryMB=10978 2024-12-04T10:09:09,244 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-04T10:09:09,326 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T10:09:09,339 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/cluster_952bd1e7-3249-0c11-0286-9464a95983a8, deleteOnExit=true 2024-12-04T10:09:09,448 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/cluster_952bd1e7-3249-0c11-0286-9464a95983a8/zookeeper_0, clientPort=58820, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/cluster_952bd1e7-3249-0c11-0286-9464a95983a8/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/cluster_952bd1e7-3249-0c11-0286-9464a95983a8/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-04T10:09:09,459 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58820 2024-12-04T10:09:09,460 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T10:09:09,537 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee with version=8 2024-12-04T10:09:09,538 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/hbase-staging 2024-12-04T10:09:09,617 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-04T10:09:09,831 INFO [Time-limited test {}] client.ConnectionUtils(128): master/84486a41f81c:0 server-side Connection retries=45 2024-12-04T10:09:09,839 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T10:09:09,840 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T10:09:09,846 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T10:09:09,847 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T10:09:09,847 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T10:09:09,957 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-04T10:09:10,032 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T10:09:10,065 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-04T10:09:10,074 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-04T10:09:10,077 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T10:09:10,109 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 117026 (auto-detected) 2024-12-04T10:09:10,110 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-04T10:09:10,129 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37539 2024-12-04T10:09:10,155 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37539 connecting to ZooKeeper ensemble=127.0.0.1:58820 2024-12-04T10:09:10,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:375390x0, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T10:09:10,230 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37539-0x101a11ac1ab0000 connected 2024-12-04T10:09:10,324 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T10:09:10,325 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T10:09:10,334 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37539-0x101a11ac1ab0000, quorum=127.0.0.1:58820, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T10:09:10,338 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee, hbase.cluster.distributed=false 2024-12-04T10:09:10,359 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37539-0x101a11ac1ab0000, quorum=127.0.0.1:58820, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T10:09:10,363 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37539 2024-12-04T10:09:10,364 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37539 2024-12-04T10:09:10,364 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37539 2024-12-04T10:09:10,365 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37539 2024-12-04T10:09:10,365 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37539 2024-12-04T10:09:10,434 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/84486a41f81c:0 server-side Connection retries=45 2024-12-04T10:09:10,435 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T10:09:10,436 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T10:09:10,436 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T10:09:10,436 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T10:09:10,436 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T10:09:10,439 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T10:09:10,441 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T10:09:10,442 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40851 2024-12-04T10:09:10,443 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40851 connecting to ZooKeeper ensemble=127.0.0.1:58820 2024-12-04T10:09:10,444 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T10:09:10,444 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T10:09:10,454 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:408510x0, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T10:09:10,455 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40851-0x101a11ac1ab0001 connected 2024-12-04T10:09:10,455 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40851-0x101a11ac1ab0001, quorum=127.0.0.1:58820, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T10:09:10,460 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T10:09:10,472 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/bucket.cache 2024-12-04T10:09:10,477 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-04T10:09:10,479 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-04T10:09:10,480 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-04T10:09:10,482 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-04T10:09:10,483 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-04T10:09:10,487 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-04T10:09:10,492 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T10:09:10,493 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-04T10:09:10,494 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/bucket.persistence is complete. 2024-12-04T10:09:10,495 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40851-0x101a11ac1ab0001, quorum=127.0.0.1:58820, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T10:09:10,501 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40851-0x101a11ac1ab0001, quorum=127.0.0.1:58820, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T10:09:10,501 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40851 2024-12-04T10:09:10,503 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40851 2024-12-04T10:09:10,506 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40851 2024-12-04T10:09:10,507 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40851 2024-12-04T10:09:10,508 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40851 2024-12-04T10:09:10,524 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/84486a41f81c:0 server-side Connection retries=45 2024-12-04T10:09:10,525 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T10:09:10,525 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T10:09:10,525 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T10:09:10,525 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T10:09:10,526 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T10:09:10,526 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T10:09:10,526 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T10:09:10,527 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46729 2024-12-04T10:09:10,529 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46729 connecting to ZooKeeper ensemble=127.0.0.1:58820 2024-12-04T10:09:10,529 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T10:09:10,530 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T10:09:10,537 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:467290x0, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T10:09:10,538 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:467290x0, quorum=127.0.0.1:58820, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T10:09:10,539 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46729-0x101a11ac1ab0002 connected 2024-12-04T10:09:10,539 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T10:09:10,540 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/bucket.cache 2024-12-04T10:09:10,541 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-04T10:09:10,541 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-04T10:09:10,541 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-04T10:09:10,543 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-04T10:09:10,543 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-04T10:09:10,543 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-04T10:09:10,544 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/bucket.persistence is complete. 2024-12-04T10:09:10,546 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-04T10:09:10,547 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T10:09:10,549 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46729-0x101a11ac1ab0002, quorum=127.0.0.1:58820, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T10:09:10,551 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46729-0x101a11ac1ab0002, quorum=127.0.0.1:58820, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T10:09:10,557 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46729 2024-12-04T10:09:10,558 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46729 2024-12-04T10:09:10,559 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46729 2024-12-04T10:09:10,562 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46729 2024-12-04T10:09:10,563 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46729 2024-12-04T10:09:10,588 DEBUG [M:0;84486a41f81c:37539 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;84486a41f81c:37539 2024-12-04T10:09:10,589 INFO [master/84486a41f81c:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/84486a41f81c,37539,1733306949675 2024-12-04T10:09:10,605 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46729-0x101a11ac1ab0002, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T10:09:10,605 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37539-0x101a11ac1ab0000, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T10:09:10,605 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40851-0x101a11ac1ab0001, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T10:09:10,607 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37539-0x101a11ac1ab0000, quorum=127.0.0.1:58820, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/84486a41f81c,37539,1733306949675 2024-12-04T10:09:10,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37539-0x101a11ac1ab0000, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T10:09:10,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40851-0x101a11ac1ab0001, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T10:09:10,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46729-0x101a11ac1ab0002, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T10:09:10,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40851-0x101a11ac1ab0001, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T10:09:10,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46729-0x101a11ac1ab0002, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T10:09:10,639 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37539-0x101a11ac1ab0000, quorum=127.0.0.1:58820, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-04T10:09:10,640 INFO [master/84486a41f81c:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/84486a41f81c,37539,1733306949675 from backup master directory 2024-12-04T10:09:10,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40851-0x101a11ac1ab0001, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T10:09:10,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46729-0x101a11ac1ab0002, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T10:09:10,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37539-0x101a11ac1ab0000, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/84486a41f81c,37539,1733306949675 2024-12-04T10:09:10,653 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37539-0x101a11ac1ab0000, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T10:09:10,653 WARN [master/84486a41f81c:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T10:09:10,654 INFO [master/84486a41f81c:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=84486a41f81c,37539,1733306949675 2024-12-04T10:09:10,656 INFO [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-04T10:09:10,657 INFO [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-04T10:09:10,664 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/hbase.id] with ID: bc72e2f1-47af-4e67-a72f-f081a0573e6a 2024-12-04T10:09:10,665 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/.tmp/hbase.id 2024-12-04T10:09:10,670 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/hbase.id] 2024-12-04T10:09:10,703 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T10:09:10,710 INFO [master/84486a41f81c:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-04T10:09:10,735 INFO [master/84486a41f81c:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 22ms. 2024-12-04T10:09:10,746 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40851-0x101a11ac1ab0001, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T10:09:10,746 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46729-0x101a11ac1ab0002, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T10:09:10,746 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37539-0x101a11ac1ab0000, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T10:09:10,772 INFO [master/84486a41f81c:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T10:09:10,774 INFO [master/84486a41f81c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-04T10:09:10,811 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T10:09:10,816 INFO [master/84486a41f81c:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T10:09:10,864 INFO [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/MasterData/data/master/store 2024-12-04T10:09:10,882 INFO [master/84486a41f81c:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-04T10:09:10,885 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T10:09:10,887 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T10:09:10,887 INFO [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T10:09:10,887 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T10:09:10,889 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T10:09:10,889 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T10:09:10,889 INFO [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T10:09:10,890 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733306950887Disabling compacts and flushes for region at 1733306950887Disabling writes for close at 1733306950889 (+2 ms)Writing region close event to WAL at 1733306950889Closed at 1733306950889 2024-12-04T10:09:10,890 WARN [master/84486a41f81c:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/MasterData/data/master/store/.initializing 2024-12-04T10:09:10,890 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/MasterData/WALs/84486a41f81c,37539,1733306949675 2024-12-04T10:09:10,902 INFO [master/84486a41f81c:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T10:09:10,914 INFO [master/84486a41f81c:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=84486a41f81c%2C37539%2C1733306949675, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/MasterData/WALs/84486a41f81c,37539,1733306949675, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/MasterData/oldWALs, maxLogs=10 2024-12-04T10:09:10,950 INFO [master/84486a41f81c:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/MasterData/WALs/84486a41f81c,37539,1733306949675/84486a41f81c%2C37539%2C1733306949675.1733306950918 2024-12-04T10:09:10,951 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-04T10:09:10,952 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-04T10:09:10,952 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T10:09:10,956 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T10:09:10,958 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T10:09:11,004 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T10:09:11,032 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-04T10:09:11,036 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T10:09:11,037 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T10:09:11,038 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T10:09:11,042 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-04T10:09:11,042 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T10:09:11,043 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T10:09:11,043 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T10:09:11,047 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-04T10:09:11,048 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T10:09:11,048 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T10:09:11,049 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T10:09:11,053 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-04T10:09:11,053 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T10:09:11,053 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T10:09:11,054 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T10:09:11,057 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T10:09:11,057 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T10:09:11,060 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T10:09:11,061 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T10:09:11,063 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T10:09:11,065 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T10:09:11,071 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T10:09:11,072 INFO [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72628991, jitterRate=0.08225630223751068}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T10:09:11,077 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733306950971Initializing all the Stores at 1733306950972 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733306950972Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733306950973 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733306950973Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733306950973Cleaning up temporary data from old regions at 1733306951061 (+88 ms)Region opened successfully at 1733306951077 (+16 ms) 2024-12-04T10:09:11,078 INFO [master/84486a41f81c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-04T10:09:11,106 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fcccfc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=84486a41f81c/172.17.0.2:0 2024-12-04T10:09:11,137 INFO [master/84486a41f81c:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-04T10:09:11,149 INFO [master/84486a41f81c:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-04T10:09:11,149 INFO [master/84486a41f81c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-04T10:09:11,152 INFO [master/84486a41f81c:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-04T10:09:11,153 INFO [master/84486a41f81c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-04T10:09:11,159 INFO [master/84486a41f81c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-04T10:09:11,159 INFO [master/84486a41f81c:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-04T10:09:11,182 INFO [master/84486a41f81c:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-04T10:09:11,192 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37539-0x101a11ac1ab0000, quorum=127.0.0.1:58820, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-04T10:09:11,236 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-04T10:09:11,239 INFO [master/84486a41f81c:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-04T10:09:11,242 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37539-0x101a11ac1ab0000, quorum=127.0.0.1:58820, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-04T10:09:11,252 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-04T10:09:11,255 INFO [master/84486a41f81c:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-04T10:09:11,260 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37539-0x101a11ac1ab0000, quorum=127.0.0.1:58820, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-04T10:09:11,271 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-04T10:09:11,273 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37539-0x101a11ac1ab0000, quorum=127.0.0.1:58820, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-04T10:09:11,279 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-04T10:09:11,300 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37539-0x101a11ac1ab0000, quorum=127.0.0.1:58820, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-04T10:09:11,310 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-04T10:09:11,329 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37539-0x101a11ac1ab0000, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T10:09:11,329 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40851-0x101a11ac1ab0001, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T10:09:11,329 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46729-0x101a11ac1ab0002, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T10:09:11,329 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37539-0x101a11ac1ab0000, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T10:09:11,329 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40851-0x101a11ac1ab0001, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T10:09:11,329 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46729-0x101a11ac1ab0002, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T10:09:11,332 INFO [master/84486a41f81c:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=84486a41f81c,37539,1733306949675, sessionid=0x101a11ac1ab0000, setting cluster-up flag (Was=false) 2024-12-04T10:09:11,362 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40851-0x101a11ac1ab0001, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T10:09:11,362 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46729-0x101a11ac1ab0002, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T10:09:11,362 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37539-0x101a11ac1ab0000, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T10:09:11,387 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-04T10:09:11,390 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=84486a41f81c,37539,1733306949675 2024-12-04T10:09:11,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40851-0x101a11ac1ab0001, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T10:09:11,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46729-0x101a11ac1ab0002, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T10:09:11,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37539-0x101a11ac1ab0000, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T10:09:11,437 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-04T10:09:11,441 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=84486a41f81c,37539,1733306949675 2024-12-04T10:09:11,448 INFO [master/84486a41f81c:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-04T10:09:11,470 INFO [RS:0;84486a41f81c:40851 {}] regionserver.HRegionServer(746): ClusterId : bc72e2f1-47af-4e67-a72f-f081a0573e6a 2024-12-04T10:09:11,470 INFO [RS:1;84486a41f81c:46729 {}] regionserver.HRegionServer(746): ClusterId : bc72e2f1-47af-4e67-a72f-f081a0573e6a 2024-12-04T10:09:11,472 DEBUG [RS:0;84486a41f81c:40851 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T10:09:11,472 DEBUG [RS:1;84486a41f81c:46729 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T10:09:11,481 DEBUG [RS:1;84486a41f81c:46729 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T10:09:11,481 DEBUG [RS:0;84486a41f81c:40851 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T10:09:11,481 DEBUG [RS:1;84486a41f81c:46729 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T10:09:11,481 DEBUG [RS:0;84486a41f81c:40851 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T10:09:11,489 DEBUG [RS:0;84486a41f81c:40851 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T10:09:11,489 DEBUG [RS:1;84486a41f81c:46729 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T10:09:11,490 DEBUG [RS:1;84486a41f81c:46729 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@386ac405, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=84486a41f81c/172.17.0.2:0 2024-12-04T10:09:11,490 DEBUG [RS:0;84486a41f81c:40851 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d30dba7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=84486a41f81c/172.17.0.2:0 2024-12-04T10:09:11,509 DEBUG [RS:1;84486a41f81c:46729 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;84486a41f81c:46729 2024-12-04T10:09:11,509 DEBUG [RS:0;84486a41f81c:40851 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;84486a41f81c:40851 2024-12-04T10:09:11,512 INFO [RS:0;84486a41f81c:40851 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-04T10:09:11,512 INFO [RS:1;84486a41f81c:46729 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-04T10:09:11,512 INFO [RS:0;84486a41f81c:40851 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-04T10:09:11,512 DEBUG [RS:0;84486a41f81c:40851 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-04T10:09:11,512 INFO [RS:1;84486a41f81c:46729 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-04T10:09:11,513 DEBUG [RS:1;84486a41f81c:46729 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-04T10:09:11,515 INFO [RS:0;84486a41f81c:40851 {}] regionserver.HRegionServer(2659): reportForDuty to master=84486a41f81c,37539,1733306949675 with port=40851, startcode=1733306950404 2024-12-04T10:09:11,515 INFO [RS:1;84486a41f81c:46729 {}] regionserver.HRegionServer(2659): reportForDuty to master=84486a41f81c,37539,1733306949675 with port=46729, startcode=1733306950524 2024-12-04T10:09:11,519 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-04T10:09:11,527 DEBUG [RS:0;84486a41f81c:40851 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T10:09:11,527 DEBUG [RS:1;84486a41f81c:46729 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T10:09:11,530 INFO [master/84486a41f81c:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-04T10:09:11,537 INFO [master/84486a41f81c:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-04T10:09:11,543 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 84486a41f81c,37539,1733306949675 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-04T10:09:11,550 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/84486a41f81c:0, corePoolSize=5, maxPoolSize=5 2024-12-04T10:09:11,550 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/84486a41f81c:0, corePoolSize=5, maxPoolSize=5 2024-12-04T10:09:11,550 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/84486a41f81c:0, corePoolSize=5, maxPoolSize=5 2024-12-04T10:09:11,551 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/84486a41f81c:0, corePoolSize=5, maxPoolSize=5 2024-12-04T10:09:11,551 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/84486a41f81c:0, corePoolSize=10, maxPoolSize=10 2024-12-04T10:09:11,551 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:11,551 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/84486a41f81c:0, corePoolSize=2, maxPoolSize=2 2024-12-04T10:09:11,552 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:11,554 INFO [master/84486a41f81c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733306981554 2024-12-04T10:09:11,556 INFO [master/84486a41f81c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-04T10:09:11,556 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T10:09:11,557 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-04T10:09:11,557 INFO [master/84486a41f81c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-04T10:09:11,561 INFO [master/84486a41f81c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-04T10:09:11,562 INFO [master/84486a41f81c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-04T10:09:11,562 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T10:09:11,562 INFO [master/84486a41f81c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-04T10:09:11,562 INFO [master/84486a41f81c:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-04T10:09:11,562 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-04T10:09:11,564 INFO [master/84486a41f81c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:11,570 INFO [master/84486a41f81c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-04T10:09:11,571 INFO [master/84486a41f81c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-04T10:09:11,572 INFO [master/84486a41f81c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-04T10:09:11,574 INFO [master/84486a41f81c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-04T10:09:11,575 INFO [master/84486a41f81c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-04T10:09:11,577 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/84486a41f81c:0:becomeActiveMaster-HFileCleaner.large.0-1733306951576,5,FailOnTimeoutGroup] 2024-12-04T10:09:11,577 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/84486a41f81c:0:becomeActiveMaster-HFileCleaner.small.0-1733306951577,5,FailOnTimeoutGroup] 2024-12-04T10:09:11,578 INFO [master/84486a41f81c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:11,578 INFO [master/84486a41f81c:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-04T10:09:11,579 INFO [master/84486a41f81c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:11,580 INFO [master/84486a41f81c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:11,589 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-04T10:09:11,590 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee 2024-12-04T10:09:11,596 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37923, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T10:09:11,596 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58771, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T10:09:11,603 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37539 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 84486a41f81c,46729,1733306950524 2024-12-04T10:09:11,606 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T10:09:11,607 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37539 {}] master.ServerManager(517): Registering regionserver=84486a41f81c,46729,1733306950524 2024-12-04T10:09:11,610 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-04T10:09:11,617 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37539 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 84486a41f81c,40851,1733306950404 2024-12-04T10:09:11,618 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37539 {}] master.ServerManager(517): Registering regionserver=84486a41f81c,40851,1733306950404 2024-12-04T10:09:11,619 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T10:09:11,619 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T10:09:11,620 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T10:09:11,620 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-04T10:09:11,622 DEBUG [RS:1;84486a41f81c:46729 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee 2024-12-04T10:09:11,622 DEBUG [RS:1;84486a41f81c:46729 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-04T10:09:11,622 DEBUG [RS:1;84486a41f81c:46729 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-04T10:09:11,623 DEBUG [RS:0;84486a41f81c:40851 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee 2024-12-04T10:09:11,623 DEBUG [RS:0;84486a41f81c:40851 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-04T10:09:11,623 DEBUG [RS:0;84486a41f81c:40851 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-04T10:09:11,625 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T10:09:11,625 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T10:09:11,625 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T10:09:11,626 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-04T10:09:11,631 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T10:09:11,631 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T10:09:11,631 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T10:09:11,631 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-04T10:09:11,635 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T10:09:11,636 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T10:09:11,636 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T10:09:11,636 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T10:09:11,638 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740 2024-12-04T10:09:11,638 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740 2024-12-04T10:09:11,638 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T10:09:11,639 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T10:09:11,639 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T10:09:11,640 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T10:09:11,646 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T10:09:11,646 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66074623, jitterRate=-0.015411391854286194}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T10:09:11,649 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733306951606Initializing all the Stores at 1733306951606Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733306951606Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733306951610 (+4 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733306951610Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733306951610Cleaning up temporary data from old regions at 1733306951639 (+29 ms)Region opened successfully at 1733306951649 (+10 ms) 2024-12-04T10:09:11,649 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T10:09:11,649 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T10:09:11,649 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T10:09:11,649 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T10:09:11,650 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T10:09:11,651 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T10:09:11,651 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733306951649Disabling compacts and flushes for region at 1733306951649Disabling writes for close at 1733306951649Writing region close event to WAL at 1733306951651 (+2 ms)Closed at 1733306951651 2024-12-04T10:09:11,653 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T10:09:11,653 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-04T10:09:11,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37539-0x101a11ac1ab0000, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T10:09:11,657 DEBUG [RS:1;84486a41f81c:46729 {}] zookeeper.ZKUtil(111): regionserver:46729-0x101a11ac1ab0002, quorum=127.0.0.1:58820, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/84486a41f81c,46729,1733306950524 2024-12-04T10:09:11,657 WARN [RS:1;84486a41f81c:46729 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T10:09:11,657 INFO [RS:1;84486a41f81c:46729 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T10:09:11,658 DEBUG [RS:0;84486a41f81c:40851 {}] zookeeper.ZKUtil(111): regionserver:40851-0x101a11ac1ab0001, quorum=127.0.0.1:58820, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/84486a41f81c,40851,1733306950404 2024-12-04T10:09:11,658 DEBUG [RS:1;84486a41f81c:46729 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/WALs/84486a41f81c,46729,1733306950524 2024-12-04T10:09:11,658 WARN [RS:0;84486a41f81c:40851 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T10:09:11,658 INFO [RS:0;84486a41f81c:40851 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T10:09:11,658 DEBUG [RS:0;84486a41f81c:40851 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/WALs/84486a41f81c,40851,1733306950404 2024-12-04T10:09:11,660 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [84486a41f81c,46729,1733306950524] 2024-12-04T10:09:11,660 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [84486a41f81c,40851,1733306950404] 2024-12-04T10:09:11,662 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-04T10:09:11,671 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T10:09:11,674 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-04T10:09:11,687 INFO [RS:0;84486a41f81c:40851 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T10:09:11,687 INFO [RS:1;84486a41f81c:46729 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T10:09:11,705 INFO [RS:1;84486a41f81c:46729 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T10:09:11,705 INFO [RS:0;84486a41f81c:40851 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T10:09:11,709 INFO [RS:1;84486a41f81c:46729 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T10:09:11,709 INFO [RS:0;84486a41f81c:40851 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T10:09:11,710 INFO [RS:1;84486a41f81c:46729 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:11,710 INFO [RS:0;84486a41f81c:40851 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:11,711 INFO [RS:0;84486a41f81c:40851 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-04T10:09:11,711 INFO [RS:1;84486a41f81c:46729 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-04T10:09:11,716 INFO [RS:1;84486a41f81c:46729 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-04T10:09:11,716 INFO [RS:0;84486a41f81c:40851 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-04T10:09:11,718 INFO [RS:1;84486a41f81c:46729 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:11,718 INFO [RS:0;84486a41f81c:40851 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:11,719 DEBUG [RS:1;84486a41f81c:46729 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:11,719 DEBUG [RS:0;84486a41f81c:40851 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:11,719 DEBUG [RS:0;84486a41f81c:40851 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:11,719 DEBUG [RS:1;84486a41f81c:46729 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:11,719 DEBUG [RS:0;84486a41f81c:40851 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:11,719 DEBUG [RS:1;84486a41f81c:46729 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:11,720 DEBUG [RS:1;84486a41f81c:46729 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:11,720 DEBUG [RS:0;84486a41f81c:40851 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:11,720 DEBUG [RS:1;84486a41f81c:46729 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:11,720 DEBUG [RS:0;84486a41f81c:40851 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:11,720 DEBUG [RS:0;84486a41f81c:40851 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/84486a41f81c:0, corePoolSize=2, maxPoolSize=2 2024-12-04T10:09:11,720 DEBUG [RS:1;84486a41f81c:46729 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/84486a41f81c:0, corePoolSize=2, maxPoolSize=2 2024-12-04T10:09:11,720 DEBUG [RS:1;84486a41f81c:46729 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:11,720 DEBUG [RS:0;84486a41f81c:40851 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:11,721 DEBUG [RS:0;84486a41f81c:40851 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:11,721 DEBUG [RS:1;84486a41f81c:46729 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:11,721 DEBUG [RS:1;84486a41f81c:46729 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:11,721 DEBUG [RS:0;84486a41f81c:40851 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:11,721 DEBUG [RS:0;84486a41f81c:40851 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:11,721 DEBUG [RS:1;84486a41f81c:46729 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:11,721 DEBUG [RS:0;84486a41f81c:40851 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:11,721 DEBUG [RS:1;84486a41f81c:46729 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:11,721 DEBUG [RS:1;84486a41f81c:46729 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:11,721 DEBUG [RS:0;84486a41f81c:40851 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:11,721 DEBUG [RS:1;84486a41f81c:46729 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/84486a41f81c:0, corePoolSize=3, maxPoolSize=3 2024-12-04T10:09:11,721 DEBUG [RS:0;84486a41f81c:40851 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/84486a41f81c:0, corePoolSize=3, maxPoolSize=3 2024-12-04T10:09:11,722 DEBUG [RS:1;84486a41f81c:46729 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/84486a41f81c:0, corePoolSize=3, maxPoolSize=3 2024-12-04T10:09:11,722 DEBUG [RS:0;84486a41f81c:40851 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/84486a41f81c:0, corePoolSize=3, maxPoolSize=3 2024-12-04T10:09:11,723 INFO [RS:1;84486a41f81c:46729 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:11,723 INFO [RS:1;84486a41f81c:46729 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:11,723 INFO [RS:1;84486a41f81c:46729 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:11,723 INFO [RS:1;84486a41f81c:46729 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:11,724 INFO [RS:1;84486a41f81c:46729 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:11,724 INFO [RS:1;84486a41f81c:46729 {}] hbase.ChoreService(168): Chore ScheduledChore name=84486a41f81c,46729,1733306950524-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T10:09:11,724 INFO [RS:0;84486a41f81c:40851 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:11,724 INFO [RS:0;84486a41f81c:40851 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:11,724 INFO [RS:0;84486a41f81c:40851 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:11,724 INFO [RS:0;84486a41f81c:40851 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:11,724 INFO [RS:0;84486a41f81c:40851 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:11,724 INFO [RS:0;84486a41f81c:40851 {}] hbase.ChoreService(168): Chore ScheduledChore name=84486a41f81c,40851,1733306950404-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T10:09:11,743 INFO [RS:1;84486a41f81c:46729 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T10:09:11,743 INFO [RS:0;84486a41f81c:40851 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T10:09:11,745 INFO [RS:1;84486a41f81c:46729 {}] hbase.ChoreService(168): Chore ScheduledChore name=84486a41f81c,46729,1733306950524-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:11,745 INFO [RS:0;84486a41f81c:40851 {}] hbase.ChoreService(168): Chore ScheduledChore name=84486a41f81c,40851,1733306950404-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:11,746 INFO [RS:1;84486a41f81c:46729 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:11,746 INFO [RS:0;84486a41f81c:40851 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:11,746 INFO [RS:1;84486a41f81c:46729 {}] regionserver.Replication(171): 84486a41f81c,46729,1733306950524 started 2024-12-04T10:09:11,746 INFO [RS:0;84486a41f81c:40851 {}] regionserver.Replication(171): 84486a41f81c,40851,1733306950404 started 2024-12-04T10:09:11,764 INFO [RS:0;84486a41f81c:40851 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:11,764 INFO [RS:1;84486a41f81c:46729 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:11,765 INFO [RS:1;84486a41f81c:46729 {}] regionserver.HRegionServer(1482): Serving as 84486a41f81c,46729,1733306950524, RpcServer on 84486a41f81c/172.17.0.2:46729, sessionid=0x101a11ac1ab0002 2024-12-04T10:09:11,765 INFO [RS:0;84486a41f81c:40851 {}] regionserver.HRegionServer(1482): Serving as 84486a41f81c,40851,1733306950404, RpcServer on 84486a41f81c/172.17.0.2:40851, sessionid=0x101a11ac1ab0001 2024-12-04T10:09:11,765 DEBUG [RS:1;84486a41f81c:46729 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T10:09:11,765 DEBUG [RS:0;84486a41f81c:40851 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T10:09:11,765 DEBUG [RS:1;84486a41f81c:46729 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 84486a41f81c,46729,1733306950524 2024-12-04T10:09:11,765 DEBUG [RS:0;84486a41f81c:40851 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 84486a41f81c,40851,1733306950404 2024-12-04T10:09:11,766 DEBUG [RS:1;84486a41f81c:46729 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '84486a41f81c,46729,1733306950524' 2024-12-04T10:09:11,766 DEBUG [RS:0;84486a41f81c:40851 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '84486a41f81c,40851,1733306950404' 2024-12-04T10:09:11,766 DEBUG [RS:1;84486a41f81c:46729 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T10:09:11,766 DEBUG [RS:0;84486a41f81c:40851 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T10:09:11,767 DEBUG [RS:1;84486a41f81c:46729 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T10:09:11,767 DEBUG [RS:0;84486a41f81c:40851 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T10:09:11,767 DEBUG [RS:1;84486a41f81c:46729 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T10:09:11,767 DEBUG [RS:0;84486a41f81c:40851 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T10:09:11,767 DEBUG [RS:1;84486a41f81c:46729 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T10:09:11,767 DEBUG [RS:0;84486a41f81c:40851 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T10:09:11,768 DEBUG [RS:1;84486a41f81c:46729 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 84486a41f81c,46729,1733306950524 2024-12-04T10:09:11,768 DEBUG [RS:0;84486a41f81c:40851 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 84486a41f81c,40851,1733306950404 2024-12-04T10:09:11,768 DEBUG [RS:1;84486a41f81c:46729 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '84486a41f81c,46729,1733306950524' 2024-12-04T10:09:11,768 DEBUG [RS:1;84486a41f81c:46729 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T10:09:11,768 DEBUG [RS:0;84486a41f81c:40851 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '84486a41f81c,40851,1733306950404' 2024-12-04T10:09:11,768 DEBUG [RS:0;84486a41f81c:40851 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T10:09:11,768 DEBUG [RS:1;84486a41f81c:46729 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T10:09:11,768 DEBUG [RS:0;84486a41f81c:40851 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T10:09:11,769 DEBUG [RS:1;84486a41f81c:46729 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T10:09:11,769 DEBUG [RS:0;84486a41f81c:40851 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T10:09:11,769 INFO [RS:1;84486a41f81c:46729 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T10:09:11,769 INFO [RS:0;84486a41f81c:40851 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T10:09:11,769 INFO [RS:1;84486a41f81c:46729 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T10:09:11,769 INFO [RS:0;84486a41f81c:40851 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T10:09:11,825 WARN [84486a41f81c:37539 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T10:09:11,877 INFO [RS:1;84486a41f81c:46729 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T10:09:11,877 INFO [RS:0;84486a41f81c:40851 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T10:09:11,879 INFO [RS:1;84486a41f81c:46729 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=84486a41f81c%2C46729%2C1733306950524, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/WALs/84486a41f81c,46729,1733306950524, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/oldWALs, maxLogs=55 2024-12-04T10:09:11,879 INFO [RS:0;84486a41f81c:40851 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=84486a41f81c%2C40851%2C1733306950404, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/WALs/84486a41f81c,40851,1733306950404, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/oldWALs, maxLogs=55 2024-12-04T10:09:11,899 INFO [RS:0;84486a41f81c:40851 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/WALs/84486a41f81c,40851,1733306950404/84486a41f81c%2C40851%2C1733306950404.1733306951883 2024-12-04T10:09:11,899 INFO [RS:1;84486a41f81c:46729 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/WALs/84486a41f81c,46729,1733306950524/84486a41f81c%2C46729%2C1733306950524.1733306951883 2024-12-04T10:09:11,899 DEBUG [RS:0;84486a41f81c:40851 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-04T10:09:11,899 DEBUG [RS:1;84486a41f81c:46729 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-04T10:09:12,078 DEBUG [84486a41f81c:37539 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-04T10:09:12,092 DEBUG [84486a41f81c:37539 {}] balancer.BalancerClusterState(204): Hosts are {84486a41f81c=0} racks are {/default-rack=0} 2024-12-04T10:09:12,095 DEBUG [84486a41f81c:37539 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-04T10:09:12,095 DEBUG [84486a41f81c:37539 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-04T10:09:12,095 DEBUG [84486a41f81c:37539 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-04T10:09:12,095 DEBUG [84486a41f81c:37539 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-04T10:09:12,095 INFO [84486a41f81c:37539 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-04T10:09:12,095 INFO [84486a41f81c:37539 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-04T10:09:12,095 DEBUG [84486a41f81c:37539 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-04T10:09:12,101 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=84486a41f81c,40851,1733306950404 2024-12-04T10:09:12,106 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 84486a41f81c,40851,1733306950404, state=OPENING 2024-12-04T10:09:12,152 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-04T10:09:12,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46729-0x101a11ac1ab0002, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T10:09:12,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37539-0x101a11ac1ab0000, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T10:09:12,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40851-0x101a11ac1ab0001, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T10:09:12,165 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T10:09:12,165 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T10:09:12,165 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T10:09:12,169 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T10:09:12,173 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=84486a41f81c,40851,1733306950404}] 2024-12-04T10:09:12,347 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T10:09:12,350 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37715, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T10:09:12,358 INFO [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-04T10:09:12,359 INFO [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T10:09:12,360 INFO [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-04T10:09:12,361 INFO [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=84486a41f81c%2C40851%2C1733306950404.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/WALs/84486a41f81c,40851,1733306950404, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/oldWALs, maxLogs=55 2024-12-04T10:09:12,378 INFO [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/WALs/84486a41f81c,40851,1733306950404/84486a41f81c%2C40851%2C1733306950404.meta.1733306952362.meta 2024-12-04T10:09:12,378 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-04T10:09:12,378 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-04T10:09:12,380 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-04T10:09:12,382 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-04T10:09:12,386 INFO [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-04T10:09:12,390 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-04T10:09:12,391 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T10:09:12,391 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-04T10:09:12,391 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-04T10:09:12,392 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-04T10:09:12,393 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T10:09:12,393 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T10:09:12,394 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T10:09:12,394 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-04T10:09:12,395 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T10:09:12,395 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T10:09:12,395 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T10:09:12,395 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-04T10:09:12,396 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T10:09:12,396 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T10:09:12,396 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T10:09:12,396 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-04T10:09:12,397 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T10:09:12,397 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T10:09:12,397 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T10:09:12,397 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T10:09:12,398 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740 2024-12-04T10:09:12,399 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740 2024-12-04T10:09:12,399 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T10:09:12,399 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T10:09:12,400 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T10:09:12,401 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T10:09:12,401 INFO [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63560873, jitterRate=-0.05286918580532074}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T10:09:12,402 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-04T10:09:12,403 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733306952391Writing region info on filesystem at 1733306952392 (+1 ms)Initializing all the Stores at 1733306952392Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733306952392Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733306952392Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733306952392Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733306952392Cleaning up temporary data from old regions at 1733306952399 (+7 ms)Running coprocessor post-open hooks at 1733306952402 (+3 ms)Region opened successfully at 1733306952403 (+1 ms) 2024-12-04T10:09:12,408 INFO [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733306952339 2024-12-04T10:09:12,416 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-04T10:09:12,416 INFO [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-04T10:09:12,417 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=84486a41f81c,40851,1733306950404 2024-12-04T10:09:12,418 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 84486a41f81c,40851,1733306950404, state=OPEN 2024-12-04T10:09:12,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37539-0x101a11ac1ab0000, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T10:09:12,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46729-0x101a11ac1ab0002, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T10:09:12,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40851-0x101a11ac1ab0001, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T10:09:12,429 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T10:09:12,429 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T10:09:12,429 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T10:09:12,430 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=84486a41f81c,40851,1733306950404 2024-12-04T10:09:12,434 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-04T10:09:12,435 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=84486a41f81c,40851,1733306950404 in 257 msec 2024-12-04T10:09:12,440 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-04T10:09:12,440 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 775 msec 2024-12-04T10:09:12,441 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T10:09:12,441 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-04T10:09:12,458 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T10:09:12,459 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=84486a41f81c,40851,1733306950404, seqNum=-1] 2024-12-04T10:09:12,476 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T10:09:12,478 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55993, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T10:09:12,494 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0160 sec 2024-12-04T10:09:12,494 INFO [master/84486a41f81c:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733306952494, completionTime=-1 2024-12-04T10:09:12,497 INFO [master/84486a41f81c:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-04T10:09:12,497 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-04T10:09:12,520 INFO [master/84486a41f81c:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-04T10:09:12,521 INFO [master/84486a41f81c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733307012521 2024-12-04T10:09:12,521 INFO [master/84486a41f81c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733307072521 2024-12-04T10:09:12,521 INFO [master/84486a41f81c:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 23 msec 2024-12-04T10:09:12,523 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-04T10:09:12,529 INFO [master/84486a41f81c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=84486a41f81c,37539,1733306949675-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:12,530 INFO [master/84486a41f81c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=84486a41f81c,37539,1733306949675-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:12,530 INFO [master/84486a41f81c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=84486a41f81c,37539,1733306949675-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:12,531 INFO [master/84486a41f81c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-84486a41f81c:37539, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:12,531 INFO [master/84486a41f81c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:12,532 INFO [master/84486a41f81c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:12,537 DEBUG [master/84486a41f81c:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-04T10:09:12,562 INFO [master/84486a41f81c:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.908sec 2024-12-04T10:09:12,564 INFO [master/84486a41f81c:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-04T10:09:12,565 INFO [master/84486a41f81c:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-04T10:09:12,565 INFO [master/84486a41f81c:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-04T10:09:12,566 INFO [master/84486a41f81c:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-04T10:09:12,566 INFO [master/84486a41f81c:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-04T10:09:12,567 INFO [master/84486a41f81c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=84486a41f81c,37539,1733306949675-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T10:09:12,567 INFO [master/84486a41f81c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=84486a41f81c,37539,1733306949675-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-04T10:09:12,569 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-04T10:09:12,569 INFO [master/84486a41f81c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-04T10:09:12,570 INFO [master/84486a41f81c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=84486a41f81c,37539,1733306949675-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:12,582 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@109a9e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T10:09:12,596 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-04T10:09:12,596 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-04T10:09:12,602 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 84486a41f81c,37539,-1 for getting cluster id 2024-12-04T10:09:12,608 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T10:09:12,616 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bc72e2f1-47af-4e67-a72f-f081a0573e6a' 2024-12-04T10:09:12,618 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T10:09:12,618 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bc72e2f1-47af-4e67-a72f-f081a0573e6a" 2024-12-04T10:09:12,619 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@319de198, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T10:09:12,619 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [84486a41f81c,37539,-1] 2024-12-04T10:09:12,621 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T10:09:12,622 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T10:09:12,624 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41388, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T10:09:12,626 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36c35605, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T10:09:12,627 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T10:09:12,634 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=84486a41f81c,40851,1733306950404, seqNum=-1] 2024-12-04T10:09:12,634 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T10:09:12,638 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49140, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T10:09:12,661 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=84486a41f81c,37539,1733306949675 2024-12-04T10:09:12,666 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T10:09:12,670 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 84486a41f81c,37539,1733306949675 2024-12-04T10:09:12,673 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@12b12a2a 2024-12-04T10:09:12,674 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T10:09:12,677 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41390, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T10:09:12,682 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37539 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1733306952661', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T10:09:12,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37539 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1733306952661 2024-12-04T10:09:12,690 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733306952661 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T10:09:12,692 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37539 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1733306952661" procId is: 4 2024-12-04T10:09:12,692 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T10:09:12,694 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733306952661 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T10:09:12,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T10:09:12,704 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733306952661-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2a1a1c56ffa4e926628a5dd6cb93f7b9, NAME => 'testBlockEvictionOnRegionMove1733306952661,,1733306952678.2a1a1c56ffa4e926628a5dd6cb93f7b9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1733306952661', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee 2024-12-04T10:09:12,710 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733306952661-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733306952661,,1733306952678.2a1a1c56ffa4e926628a5dd6cb93f7b9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T10:09:12,710 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733306952661-pool-0 {}] regionserver.HRegion(1722): Closing 2a1a1c56ffa4e926628a5dd6cb93f7b9, disabling compactions & flushes 2024-12-04T10:09:12,710 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733306952661-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733306952661,,1733306952678.2a1a1c56ffa4e926628a5dd6cb93f7b9. 2024-12-04T10:09:12,710 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733306952661-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733306952661,,1733306952678.2a1a1c56ffa4e926628a5dd6cb93f7b9. 2024-12-04T10:09:12,710 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733306952661-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733306952661,,1733306952678.2a1a1c56ffa4e926628a5dd6cb93f7b9. after waiting 0 ms 2024-12-04T10:09:12,710 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733306952661-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733306952661,,1733306952678.2a1a1c56ffa4e926628a5dd6cb93f7b9. 2024-12-04T10:09:12,710 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733306952661-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733306952661,,1733306952678.2a1a1c56ffa4e926628a5dd6cb93f7b9. 2024-12-04T10:09:12,710 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733306952661-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2a1a1c56ffa4e926628a5dd6cb93f7b9: Waiting for close lock at 1733306952710Disabling compacts and flushes for region at 1733306952710Disabling writes for close at 1733306952710Writing region close event to WAL at 1733306952710Closed at 1733306952710 2024-12-04T10:09:12,712 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733306952661 execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T10:09:12,717 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1733306952661,,1733306952678.2a1a1c56ffa4e926628a5dd6cb93f7b9.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733306952712"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733306952712"}]},"ts":"1733306952712"} 2024-12-04T10:09:12,722 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T10:09:12,723 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733306952661 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T10:09:12,726 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1733306952661","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733306952724"}]},"ts":"1733306952724"} 2024-12-04T10:09:12,730 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1733306952661, state=ENABLING in hbase:meta 2024-12-04T10:09:12,730 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {84486a41f81c=0} racks are {/default-rack=0} 2024-12-04T10:09:12,731 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-04T10:09:12,731 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-04T10:09:12,731 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-04T10:09:12,731 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-04T10:09:12,731 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-04T10:09:12,731 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-04T10:09:12,731 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-04T10:09:12,733 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733306952661, region=2a1a1c56ffa4e926628a5dd6cb93f7b9, ASSIGN}] 2024-12-04T10:09:12,735 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733306952661, region=2a1a1c56ffa4e926628a5dd6cb93f7b9, ASSIGN 2024-12-04T10:09:12,736 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733306952661, region=2a1a1c56ffa4e926628a5dd6cb93f7b9, ASSIGN; state=OFFLINE, location=84486a41f81c,40851,1733306950404; forceNewPlan=false, retain=false 2024-12-04T10:09:12,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T10:09:12,889 INFO [84486a41f81c:37539 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-04T10:09:12,891 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=2a1a1c56ffa4e926628a5dd6cb93f7b9, regionState=OPENING, regionLocation=84486a41f81c,40851,1733306950404 2024-12-04T10:09:12,895 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733306952661, region=2a1a1c56ffa4e926628a5dd6cb93f7b9, ASSIGN because future has completed 2024-12-04T10:09:12,896 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2a1a1c56ffa4e926628a5dd6cb93f7b9, server=84486a41f81c,40851,1733306950404}] 2024-12-04T10:09:13,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T10:09:13,055 INFO [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1733306952661,,1733306952678.2a1a1c56ffa4e926628a5dd6cb93f7b9. 2024-12-04T10:09:13,056 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 2a1a1c56ffa4e926628a5dd6cb93f7b9, NAME => 'testBlockEvictionOnRegionMove1733306952661,,1733306952678.2a1a1c56ffa4e926628a5dd6cb93f7b9.', STARTKEY => '', ENDKEY => ''} 2024-12-04T10:09:13,056 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1733306952661 2a1a1c56ffa4e926628a5dd6cb93f7b9 2024-12-04T10:09:13,057 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733306952661,,1733306952678.2a1a1c56ffa4e926628a5dd6cb93f7b9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T10:09:13,057 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 2a1a1c56ffa4e926628a5dd6cb93f7b9 2024-12-04T10:09:13,057 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 2a1a1c56ffa4e926628a5dd6cb93f7b9 2024-12-04T10:09:13,058 INFO [StoreOpener-2a1a1c56ffa4e926628a5dd6cb93f7b9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 2a1a1c56ffa4e926628a5dd6cb93f7b9 2024-12-04T10:09:13,063 INFO [StoreOpener-2a1a1c56ffa4e926628a5dd6cb93f7b9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2a1a1c56ffa4e926628a5dd6cb93f7b9 columnFamilyName family 2024-12-04T10:09:13,064 DEBUG [StoreOpener-2a1a1c56ffa4e926628a5dd6cb93f7b9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T10:09:13,064 INFO [StoreOpener-2a1a1c56ffa4e926628a5dd6cb93f7b9-1 {}] regionserver.HStore(327): Store=2a1a1c56ffa4e926628a5dd6cb93f7b9/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T10:09:13,064 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 2a1a1c56ffa4e926628a5dd6cb93f7b9 2024-12-04T10:09:13,065 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/default/testBlockEvictionOnRegionMove1733306952661/2a1a1c56ffa4e926628a5dd6cb93f7b9 2024-12-04T10:09:13,065 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/default/testBlockEvictionOnRegionMove1733306952661/2a1a1c56ffa4e926628a5dd6cb93f7b9 2024-12-04T10:09:13,065 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 2a1a1c56ffa4e926628a5dd6cb93f7b9 2024-12-04T10:09:13,066 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 2a1a1c56ffa4e926628a5dd6cb93f7b9 2024-12-04T10:09:13,066 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 2a1a1c56ffa4e926628a5dd6cb93f7b9 2024-12-04T10:09:13,072 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/default/testBlockEvictionOnRegionMove1733306952661/2a1a1c56ffa4e926628a5dd6cb93f7b9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T10:09:13,072 INFO [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 2a1a1c56ffa4e926628a5dd6cb93f7b9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71480953, jitterRate=0.0651492029428482}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T10:09:13,072 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2a1a1c56ffa4e926628a5dd6cb93f7b9 2024-12-04T10:09:13,073 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 2a1a1c56ffa4e926628a5dd6cb93f7b9: Running coprocessor pre-open hook at 1733306953057Writing region info on filesystem at 1733306953057Initializing all the Stores at 1733306953058 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733306953058Cleaning up temporary data from old regions at 1733306953066 (+8 ms)Running coprocessor post-open hooks at 1733306953072 (+6 ms)Region opened successfully at 1733306953073 (+1 ms) 2024-12-04T10:09:13,074 INFO [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1733306952661,,1733306952678.2a1a1c56ffa4e926628a5dd6cb93f7b9., pid=6, masterSystemTime=1733306953050 2024-12-04T10:09:13,077 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1733306952661,,1733306952678.2a1a1c56ffa4e926628a5dd6cb93f7b9. 2024-12-04T10:09:13,077 INFO [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1733306952661,,1733306952678.2a1a1c56ffa4e926628a5dd6cb93f7b9. 2024-12-04T10:09:13,077 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=2a1a1c56ffa4e926628a5dd6cb93f7b9, regionState=OPEN, openSeqNum=2, regionLocation=84486a41f81c,40851,1733306950404 2024-12-04T10:09:13,080 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2a1a1c56ffa4e926628a5dd6cb93f7b9, server=84486a41f81c,40851,1733306950404 because future has completed 2024-12-04T10:09:13,083 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-04T10:09:13,084 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 2a1a1c56ffa4e926628a5dd6cb93f7b9, server=84486a41f81c,40851,1733306950404 in 185 msec 2024-12-04T10:09:13,086 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-04T10:09:13,086 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733306952661, region=2a1a1c56ffa4e926628a5dd6cb93f7b9, ASSIGN in 352 msec 2024-12-04T10:09:13,087 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733306952661 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T10:09:13,087 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1733306952661","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733306953087"}]},"ts":"1733306953087"} 2024-12-04T10:09:13,090 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1733306952661, state=ENABLED in hbase:meta 2024-12-04T10:09:13,091 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733306952661 execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T10:09:13,093 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1733306952661 in 406 msec 2024-12-04T10:09:13,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T10:09:13,322 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1733306952661 get assigned. Timeout = 60000ms 2024-12-04T10:09:13,322 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1733306952661 completed 2024-12-04T10:09:13,325 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T10:09:13,332 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1733306952661 assigned to meta. Checking AM states. 2024-12-04T10:09:13,333 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T10:09:13,334 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1733306952661 assigned. 2024-12-04T10:09:13,344 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1733306952661', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1733306952661,,1733306952678.2a1a1c56ffa4e926628a5dd6cb93f7b9., hostname=84486a41f81c,40851,1733306950404, seqNum=2] 2024-12-04T10:09:13,357 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2a1a1c56ffa4e926628a5dd6cb93f7b9 1/1 column families, dataSize=78 B heapSize=480 B 2024-12-04T10:09:13,380 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 3a2dbb30c7fc4447b6a3faf82a4584ea 2024-12-04T10:09:13,411 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/default/testBlockEvictionOnRegionMove1733306952661/2a1a1c56ffa4e926628a5dd6cb93f7b9/.tmp/family/3a2dbb30c7fc4447b6a3faf82a4584ea is 43, key is row1/family:qf1/1/Put/seqid=0 2024-12-04T10:09:13,415 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/default/testBlockEvictionOnRegionMove1733306952661/2a1a1c56ffa4e926628a5dd6cb93f7b9/.tmp/family/3a2dbb30c7fc4447b6a3faf82a4584ea 2024-12-04T10:09:13,443 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/default/testBlockEvictionOnRegionMove1733306952661/2a1a1c56ffa4e926628a5dd6cb93f7b9/.tmp/family/3a2dbb30c7fc4447b6a3faf82a4584ea as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/default/testBlockEvictionOnRegionMove1733306952661/2a1a1c56ffa4e926628a5dd6cb93f7b9/family/3a2dbb30c7fc4447b6a3faf82a4584ea 2024-12-04T10:09:13,444 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/default/testBlockEvictionOnRegionMove1733306952661/2a1a1c56ffa4e926628a5dd6cb93f7b9/family/3a2dbb30c7fc4447b6a3faf82a4584ea, delay=958 ms 2024-12-04T10:09:13,446 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/default/testBlockEvictionOnRegionMove1733306952661/2a1a1c56ffa4e926628a5dd6cb93f7b9/family/3a2dbb30c7fc4447b6a3faf82a4584ea, entries=2, sequenceid=6, filesize=4.9 K 2024-12-04T10:09:13,451 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 2a1a1c56ffa4e926628a5dd6cb93f7b9 in 93ms, sequenceid=6, compaction requested=false 2024-12-04T10:09:13,452 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1733306952661' 2024-12-04T10:09:13,453 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 2a1a1c56ffa4e926628a5dd6cb93f7b9: 2024-12-04T10:09:14,406 DEBUG [hfile-prefetch-1733306953444 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3a2dbb30c7fc4447b6a3faf82a4584ea_0. 2024-12-04T10:09:14,414 DEBUG [hfile-prefetch-1733306953444 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3a2dbb30c7fc4447b6a3faf82a4584ea_117. 2024-12-04T10:09:14,415 DEBUG [hfile-prefetch-1733306953444 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/default/testBlockEvictionOnRegionMove1733306952661/2a1a1c56ffa4e926628a5dd6cb93f7b9/family/3a2dbb30c7fc4447b6a3faf82a4584ea, with total blocks 2, and data blocks 1 2024-12-04T10:09:14,415 DEBUG [hfile-prefetch-1733306953444 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-04T10:09:14,416 DEBUG [hfile-prefetch-1733306953444 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/default/testBlockEvictionOnRegionMove1733306952661/2a1a1c56ffa4e926628a5dd6cb93f7b9/family/3a2dbb30c7fc4447b6a3faf82a4584ea, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-12-04T10:09:14,416 DEBUG [hfile-prefetch-1733306953444 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/default/testBlockEvictionOnRegionMove1733306952661/2a1a1c56ffa4e926628a5dd6cb93f7b9/family/3a2dbb30c7fc4447b6a3faf82a4584ea. Will sleep 100ms and try the verification again. 2024-12-04T10:09:14,467 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37539 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=2a1a1c56ffa4e926628a5dd6cb93f7b9, source=84486a41f81c,40851,1733306950404, destination=84486a41f81c,46729,1733306950524, warming up region on 84486a41f81c,46729,1733306950524 2024-12-04T10:09:14,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37539 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T10:09:14,469 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37539 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=2a1a1c56ffa4e926628a5dd6cb93f7b9, source=84486a41f81c,40851,1733306950404, destination=84486a41f81c,46729,1733306950524, running balancer 2024-12-04T10:09:14,470 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43873, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T10:09:14,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37539 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733306952661, region=2a1a1c56ffa4e926628a5dd6cb93f7b9, REOPEN/MOVE 2024-12-04T10:09:14,471 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733306952661, region=2a1a1c56ffa4e926628a5dd6cb93f7b9, REOPEN/MOVE 2024-12-04T10:09:14,472 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=2a1a1c56ffa4e926628a5dd6cb93f7b9, regionState=CLOSING, regionLocation=84486a41f81c,40851,1733306950404 2024-12-04T10:09:14,472 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46729 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1733306952661,,1733306952678.2a1a1c56ffa4e926628a5dd6cb93f7b9. 2024-12-04T10:09:14,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46729 {}] regionserver.HRegion(7855): Warmup {ENCODED => 2a1a1c56ffa4e926628a5dd6cb93f7b9, NAME => 'testBlockEvictionOnRegionMove1733306952661,,1733306952678.2a1a1c56ffa4e926628a5dd6cb93f7b9.', STARTKEY => '', ENDKEY => ''} 2024-12-04T10:09:14,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46729 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733306952661,,1733306952678.2a1a1c56ffa4e926628a5dd6cb93f7b9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T10:09:14,474 INFO [StoreOpener-2a1a1c56ffa4e926628a5dd6cb93f7b9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 2a1a1c56ffa4e926628a5dd6cb93f7b9 2024-12-04T10:09:14,475 INFO [StoreOpener-2a1a1c56ffa4e926628a5dd6cb93f7b9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2a1a1c56ffa4e926628a5dd6cb93f7b9 columnFamilyName family 2024-12-04T10:09:14,475 DEBUG [StoreOpener-2a1a1c56ffa4e926628a5dd6cb93f7b9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T10:09:14,476 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733306952661, region=2a1a1c56ffa4e926628a5dd6cb93f7b9, REOPEN/MOVE because future has completed 2024-12-04T10:09:14,476 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-12-04T10:09:14,476 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2a1a1c56ffa4e926628a5dd6cb93f7b9, server=84486a41f81c,40851,1733306950404}] 2024-12-04T10:09:14,478 DEBUG [StoreOpener-2a1a1c56ffa4e926628a5dd6cb93f7b9-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/default/testBlockEvictionOnRegionMove1733306952661/2a1a1c56ffa4e926628a5dd6cb93f7b9/family/3a2dbb30c7fc4447b6a3faf82a4584ea 2024-12-04T10:09:14,478 INFO [StoreOpener-2a1a1c56ffa4e926628a5dd6cb93f7b9-1 {}] regionserver.HStore(327): Store=2a1a1c56ffa4e926628a5dd6cb93f7b9/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T10:09:14,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46729 {}] regionserver.HRegion(1722): Closing 2a1a1c56ffa4e926628a5dd6cb93f7b9, disabling compactions & flushes 2024-12-04T10:09:14,479 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46729 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733306952661,,1733306952678.2a1a1c56ffa4e926628a5dd6cb93f7b9. 2024-12-04T10:09:14,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46729 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733306952661,,1733306952678.2a1a1c56ffa4e926628a5dd6cb93f7b9. 2024-12-04T10:09:14,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46729 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733306952661,,1733306952678.2a1a1c56ffa4e926628a5dd6cb93f7b9. after waiting 0 ms 2024-12-04T10:09:14,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46729 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733306952661,,1733306952678.2a1a1c56ffa4e926628a5dd6cb93f7b9. 2024-12-04T10:09:14,480 DEBUG [StoreFileCloser-2a1a1c56ffa4e926628a5dd6cb93f7b9-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/default/testBlockEvictionOnRegionMove1733306952661/2a1a1c56ffa4e926628a5dd6cb93f7b9/family/3a2dbb30c7fc4447b6a3faf82a4584ea 2024-12-04T10:09:14,481 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46729 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733306952661,,1733306952678.2a1a1c56ffa4e926628a5dd6cb93f7b9. 2024-12-04T10:09:14,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46729 {}] regionserver.HRegion(1676): Region close journal for 2a1a1c56ffa4e926628a5dd6cb93f7b9: Waiting for close lock at 1733306954479Disabling compacts and flushes for region at 1733306954479Disabling writes for close at 1733306954479Writing region close event to WAL at 1733306954481 (+2 ms)Closed at 1733306954481 2024-12-04T10:09:14,482 WARN [hfile-prefetch-1733306953444 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/default/testBlockEvictionOnRegionMove1733306952661/2a1a1c56ffa4e926628a5dd6cb93f7b9/family/3a2dbb30c7fc4447b6a3faf82a4584ea, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-12-04T10:09:14,483 DEBUG [hfile-prefetch-1733306953444 {}] hfile.PrefetchExecutor(121): Prefetch completed for 3a2dbb30c7fc4447b6a3faf82a4584ea 2024-12-04T10:09:14,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37539 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-04T10:09:14,585 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-04T10:09:14,585 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-04T10:09:14,585 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-04T10:09:14,586 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-12-04T10:09:14,608 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-12-04T10:09:14,608 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1733306954585 2024-12-04T10:09:14,608 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-04T10:09:14,635 INFO [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 2a1a1c56ffa4e926628a5dd6cb93f7b9 2024-12-04T10:09:14,635 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-12-04T10:09:14,636 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 2a1a1c56ffa4e926628a5dd6cb93f7b9, disabling compactions & flushes 2024-12-04T10:09:14,636 INFO [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733306952661,,1733306952678.2a1a1c56ffa4e926628a5dd6cb93f7b9. 2024-12-04T10:09:14,636 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733306952661,,1733306952678.2a1a1c56ffa4e926628a5dd6cb93f7b9. 2024-12-04T10:09:14,636 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733306952661,,1733306952678.2a1a1c56ffa4e926628a5dd6cb93f7b9. after waiting 0 ms 2024-12-04T10:09:14,636 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733306952661,,1733306952678.2a1a1c56ffa4e926628a5dd6cb93f7b9. 2024-12-04T10:09:14,639 DEBUG [StoreFileCloser-2a1a1c56ffa4e926628a5dd6cb93f7b9-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file 3a2dbb30c7fc4447b6a3faf82a4584ea, starting offset: 0, end offset: 9223372036854775807 2024-12-04T10:09:14,641 DEBUG [StoreFileCloser-2a1a1c56ffa4e926628a5dd6cb93f7b9-family-1 {}] bucket.BucketCache(856): removed key 3a2dbb30c7fc4447b6a3faf82a4584ea_0 from back map with offset lock 2094080 in the evict process 2024-12-04T10:09:14,646 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-04T10:09:14,646 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-04T10:09:14,646 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-04T10:09:14,646 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-04T10:09:14,647 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/default/testBlockEvictionOnRegionMove1733306952661/2a1a1c56ffa4e926628a5dd6cb93f7b9/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-04T10:09:14,649 INFO [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733306952661,,1733306952678.2a1a1c56ffa4e926628a5dd6cb93f7b9. 2024-12-04T10:09:14,649 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 2a1a1c56ffa4e926628a5dd6cb93f7b9: Waiting for close lock at 1733306954636Running coprocessor pre-close hooks at 1733306954636Disabling compacts and flushes for region at 1733306954636Disabling writes for close at 1733306954636Writing region close event to WAL at 1733306954641 (+5 ms)Running coprocessor post-close hooks at 1733306954648 (+7 ms)Closed at 1733306954649 (+1 ms) 2024-12-04T10:09:14,649 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-04T10:09:14,649 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733306954646 2024-12-04T10:09:14,650 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-04T10:09:14,650 INFO [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 2a1a1c56ffa4e926628a5dd6cb93f7b9 move to 84486a41f81c,46729,1733306950524 record at close sequenceid=6 2024-12-04T10:09:14,653 INFO [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 2a1a1c56ffa4e926628a5dd6cb93f7b9 2024-12-04T10:09:14,654 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=2a1a1c56ffa4e926628a5dd6cb93f7b9, regionState=CLOSED 2024-12-04T10:09:14,656 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2a1a1c56ffa4e926628a5dd6cb93f7b9, server=84486a41f81c,40851,1733306950404 because future has completed 2024-12-04T10:09:14,660 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-04T10:09:14,660 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 2a1a1c56ffa4e926628a5dd6cb93f7b9, server=84486a41f81c,40851,1733306950404 in 181 msec 2024-12-04T10:09:14,661 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733306952661, region=2a1a1c56ffa4e926628a5dd6cb93f7b9, REOPEN/MOVE; state=CLOSED, location=84486a41f81c,46729,1733306950524; forceNewPlan=false, retain=false 2024-12-04T10:09:14,812 INFO [84486a41f81c:37539 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-04T10:09:14,812 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=2a1a1c56ffa4e926628a5dd6cb93f7b9, regionState=OPENING, regionLocation=84486a41f81c,46729,1733306950524 2024-12-04T10:09:14,817 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733306952661, region=2a1a1c56ffa4e926628a5dd6cb93f7b9, REOPEN/MOVE because future has completed 2024-12-04T10:09:14,818 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2a1a1c56ffa4e926628a5dd6cb93f7b9, server=84486a41f81c,46729,1733306950524}] 2024-12-04T10:09:14,974 INFO [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1733306952661,,1733306952678.2a1a1c56ffa4e926628a5dd6cb93f7b9. 2024-12-04T10:09:14,975 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 2a1a1c56ffa4e926628a5dd6cb93f7b9, NAME => 'testBlockEvictionOnRegionMove1733306952661,,1733306952678.2a1a1c56ffa4e926628a5dd6cb93f7b9.', STARTKEY => '', ENDKEY => ''} 2024-12-04T10:09:14,976 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1733306952661 2a1a1c56ffa4e926628a5dd6cb93f7b9 2024-12-04T10:09:14,976 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733306952661,,1733306952678.2a1a1c56ffa4e926628a5dd6cb93f7b9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T10:09:14,976 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 2a1a1c56ffa4e926628a5dd6cb93f7b9 2024-12-04T10:09:14,976 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 2a1a1c56ffa4e926628a5dd6cb93f7b9 2024-12-04T10:09:14,978 INFO [StoreOpener-2a1a1c56ffa4e926628a5dd6cb93f7b9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 2a1a1c56ffa4e926628a5dd6cb93f7b9 2024-12-04T10:09:14,980 INFO [StoreOpener-2a1a1c56ffa4e926628a5dd6cb93f7b9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2a1a1c56ffa4e926628a5dd6cb93f7b9 columnFamilyName family 2024-12-04T10:09:14,980 DEBUG [StoreOpener-2a1a1c56ffa4e926628a5dd6cb93f7b9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T10:09:14,983 DEBUG [StoreFileOpener-2a1a1c56ffa4e926628a5dd6cb93f7b9-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/default/testBlockEvictionOnRegionMove1733306952661/2a1a1c56ffa4e926628a5dd6cb93f7b9/family/3a2dbb30c7fc4447b6a3faf82a4584ea, delay=925 ms 2024-12-04T10:09:14,986 DEBUG [StoreOpener-2a1a1c56ffa4e926628a5dd6cb93f7b9-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/default/testBlockEvictionOnRegionMove1733306952661/2a1a1c56ffa4e926628a5dd6cb93f7b9/family/3a2dbb30c7fc4447b6a3faf82a4584ea 2024-12-04T10:09:14,986 INFO [StoreOpener-2a1a1c56ffa4e926628a5dd6cb93f7b9-1 {}] regionserver.HStore(327): Store=2a1a1c56ffa4e926628a5dd6cb93f7b9/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T10:09:14,986 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 2a1a1c56ffa4e926628a5dd6cb93f7b9 2024-12-04T10:09:14,987 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/default/testBlockEvictionOnRegionMove1733306952661/2a1a1c56ffa4e926628a5dd6cb93f7b9 2024-12-04T10:09:14,988 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/default/testBlockEvictionOnRegionMove1733306952661/2a1a1c56ffa4e926628a5dd6cb93f7b9 2024-12-04T10:09:14,988 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 2a1a1c56ffa4e926628a5dd6cb93f7b9 2024-12-04T10:09:14,988 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 2a1a1c56ffa4e926628a5dd6cb93f7b9 2024-12-04T10:09:14,993 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 2a1a1c56ffa4e926628a5dd6cb93f7b9 2024-12-04T10:09:14,994 INFO [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 2a1a1c56ffa4e926628a5dd6cb93f7b9; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66576649, jitterRate=-0.00793062150478363}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T10:09:14,994 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2a1a1c56ffa4e926628a5dd6cb93f7b9 2024-12-04T10:09:14,995 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 2a1a1c56ffa4e926628a5dd6cb93f7b9: Running coprocessor pre-open hook at 1733306954977Writing region info on filesystem at 1733306954977Initializing all the Stores at 1733306954977Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733306954977Cleaning up temporary data from old regions at 1733306954988 (+11 ms)Running coprocessor post-open hooks at 1733306954994 (+6 ms)Region opened successfully at 1733306954995 (+1 ms) 2024-12-04T10:09:14,996 INFO [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1733306952661,,1733306952678.2a1a1c56ffa4e926628a5dd6cb93f7b9., pid=9, masterSystemTime=1733306954970 2024-12-04T10:09:15,000 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1733306952661,,1733306952678.2a1a1c56ffa4e926628a5dd6cb93f7b9. 2024-12-04T10:09:15,000 INFO [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1733306952661,,1733306952678.2a1a1c56ffa4e926628a5dd6cb93f7b9. 2024-12-04T10:09:15,000 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=2a1a1c56ffa4e926628a5dd6cb93f7b9, regionState=OPEN, openSeqNum=10, regionLocation=84486a41f81c,46729,1733306950524 2024-12-04T10:09:15,003 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2a1a1c56ffa4e926628a5dd6cb93f7b9, server=84486a41f81c,46729,1733306950524 because future has completed 2024-12-04T10:09:15,007 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-04T10:09:15,007 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 2a1a1c56ffa4e926628a5dd6cb93f7b9, server=84486a41f81c,46729,1733306950524 in 187 msec 2024-12-04T10:09:15,009 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733306952661, region=2a1a1c56ffa4e926628a5dd6cb93f7b9, REOPEN/MOVE in 538 msec 2024-12-04T10:09:15,023 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-04T10:09:15,024 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T10:09:15,024 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T10:09:15,028 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T10:09:15,028 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T10:09:15,028 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T10:09:15,028 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-04T10:09:15,028 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1692727640, stopped=false 2024-12-04T10:09:15,029 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=84486a41f81c,37539,1733306949675 2024-12-04T10:09:15,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37539-0x101a11ac1ab0000, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T10:09:15,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40851-0x101a11ac1ab0001, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T10:09:15,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46729-0x101a11ac1ab0002, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T10:09:15,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37539-0x101a11ac1ab0000, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T10:09:15,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40851-0x101a11ac1ab0001, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T10:09:15,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46729-0x101a11ac1ab0002, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T10:09:15,046 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T10:09:15,047 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T10:09:15,047 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40851-0x101a11ac1ab0001, quorum=127.0.0.1:58820, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T10:09:15,047 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37539-0x101a11ac1ab0000, quorum=127.0.0.1:58820, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T10:09:15,047 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46729-0x101a11ac1ab0002, quorum=127.0.0.1:58820, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T10:09:15,048 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T10:09:15,048 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T10:09:15,048 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '84486a41f81c,40851,1733306950404' ***** 2024-12-04T10:09:15,049 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-04T10:09:15,049 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '84486a41f81c,46729,1733306950524' ***** 2024-12-04T10:09:15,049 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-04T10:09:15,049 INFO [RS:0;84486a41f81c:40851 {}] bucket.BucketCache(1738): Disabling cache 2024-12-04T10:09:15,049 INFO [RS:1;84486a41f81c:46729 {}] bucket.BucketCache(1738): Disabling cache 2024-12-04T10:09:15,050 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-04T10:09:15,050 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-04T10:09:15,050 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-04T10:09:15,050 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-04T10:09:15,050 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-04T10:09:15,050 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-04T10:09:15,050 INFO [RS:0;84486a41f81c:40851 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/bucket.persistence 2024-12-04T10:09:15,050 INFO [RS:0;84486a41f81c:40851 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-04T10:09:15,050 INFO [RS:1;84486a41f81c:46729 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/bucket.persistence 2024-12-04T10:09:15,050 INFO [RS:1;84486a41f81c:46729 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-04T10:09:15,050 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-04T10:09:15,051 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-04T10:09:15,061 DEBUG [RS:0;84486a41f81c:40851 {}] bucket.BucketCache(1487): Thread RS:0;84486a41f81c:40851 started persisting bucket cache to file 2024-12-04T10:09:15,061 DEBUG [RS:1;84486a41f81c:46729 {}] bucket.BucketCache(1487): Thread RS:1;84486a41f81c:46729 started persisting bucket cache to file 2024-12-04T10:09:15,061 DEBUG [RS:0;84486a41f81c:40851 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-04T10:09:15,061 DEBUG [RS:0;84486a41f81c:40851 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-04T10:09:15,061 DEBUG [RS:1;84486a41f81c:46729 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-04T10:09:15,061 DEBUG [RS:1;84486a41f81c:46729 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-04T10:09:15,064 DEBUG [RS:0;84486a41f81c:40851 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-04T10:09:15,064 DEBUG [RS:0;84486a41f81c:40851 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733306955061 2024-12-04T10:09:15,064 DEBUG [RS:1;84486a41f81c:46729 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-04T10:09:15,064 DEBUG [RS:1;84486a41f81c:46729 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733306955061 2024-12-04T10:09:15,064 DEBUG [RS:1;84486a41f81c:46729 {}] bucket.BucketCache(1509): Thread RS:1;84486a41f81c:46729 finished persisting bucket cache to file, renaming 2024-12-04T10:09:15,064 INFO [RS:1;84486a41f81c:46729 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T10:09:15,065 INFO [RS:1;84486a41f81c:46729 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-04T10:09:15,065 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-04T10:09:15,065 INFO [RS:1;84486a41f81c:46729 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-04T10:09:15,065 INFO [RS:1;84486a41f81c:46729 {}] regionserver.HRegionServer(3091): Received CLOSE for 2a1a1c56ffa4e926628a5dd6cb93f7b9 2024-12-04T10:09:15,065 INFO [RS:1;84486a41f81c:46729 {}] regionserver.HRegionServer(959): stopping server 84486a41f81c,46729,1733306950524 2024-12-04T10:09:15,066 INFO [RS:1;84486a41f81c:46729 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T10:09:15,066 INFO [RS:1;84486a41f81c:46729 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;84486a41f81c:46729. 2024-12-04T10:09:15,066 DEBUG [RS:1;84486a41f81c:46729 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T10:09:15,066 DEBUG [RS:1;84486a41f81c:46729 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T10:09:15,066 INFO [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1733306952661,,1733306952678.2a1a1c56ffa4e926628a5dd6cb93f7b9. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-04T10:09:15,066 INFO [RS:1;84486a41f81c:46729 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-04T10:09:15,066 DEBUG [RS:1;84486a41f81c:46729 {}] regionserver.HRegionServer(1325): Online Regions={2a1a1c56ffa4e926628a5dd6cb93f7b9=testBlockEvictionOnRegionMove1733306952661,,1733306952678.2a1a1c56ffa4e926628a5dd6cb93f7b9.} 2024-12-04T10:09:15,066 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2a1a1c56ffa4e926628a5dd6cb93f7b9, disabling compactions & flushes 2024-12-04T10:09:15,066 INFO [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733306952661,,1733306952678.2a1a1c56ffa4e926628a5dd6cb93f7b9. 2024-12-04T10:09:15,066 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733306952661,,1733306952678.2a1a1c56ffa4e926628a5dd6cb93f7b9. 2024-12-04T10:09:15,066 DEBUG [RS:1;84486a41f81c:46729 {}] regionserver.HRegionServer(1351): Waiting on 2a1a1c56ffa4e926628a5dd6cb93f7b9 2024-12-04T10:09:15,066 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733306952661,,1733306952678.2a1a1c56ffa4e926628a5dd6cb93f7b9. after waiting 0 ms 2024-12-04T10:09:15,066 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733306952661,,1733306952678.2a1a1c56ffa4e926628a5dd6cb93f7b9. 2024-12-04T10:09:15,067 DEBUG [StoreFileCloser-2a1a1c56ffa4e926628a5dd6cb93f7b9-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/default/testBlockEvictionOnRegionMove1733306952661/2a1a1c56ffa4e926628a5dd6cb93f7b9/family/3a2dbb30c7fc4447b6a3faf82a4584ea 2024-12-04T10:09:15,071 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/default/testBlockEvictionOnRegionMove1733306952661/2a1a1c56ffa4e926628a5dd6cb93f7b9/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-12-04T10:09:15,072 INFO [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733306952661,,1733306952678.2a1a1c56ffa4e926628a5dd6cb93f7b9. 2024-12-04T10:09:15,072 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2a1a1c56ffa4e926628a5dd6cb93f7b9: Waiting for close lock at 1733306955066Running coprocessor pre-close hooks at 1733306955066Disabling compacts and flushes for region at 1733306955066Disabling writes for close at 1733306955066Writing region close event to WAL at 1733306955068 (+2 ms)Running coprocessor post-close hooks at 1733306955072 (+4 ms)Closed at 1733306955072 2024-12-04T10:09:15,072 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1733306952661,,1733306952678.2a1a1c56ffa4e926628a5dd6cb93f7b9. 2024-12-04T10:09:15,101 DEBUG [RS:0;84486a41f81c:40851 {}] bucket.BucketCache(1509): Thread RS:0;84486a41f81c:40851 finished persisting bucket cache to file, renaming 2024-12-04T10:09:15,101 WARN [RS:0;84486a41f81c:40851 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-12-04T10:09:15,102 INFO [RS:0;84486a41f81c:40851 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T10:09:15,102 INFO [RS:0;84486a41f81c:40851 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-04T10:09:15,102 INFO [RS:0;84486a41f81c:40851 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-04T10:09:15,102 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-04T10:09:15,102 INFO [RS:0;84486a41f81c:40851 {}] regionserver.HRegionServer(959): stopping server 84486a41f81c,40851,1733306950404 2024-12-04T10:09:15,102 INFO [RS:0;84486a41f81c:40851 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T10:09:15,102 INFO [RS:0;84486a41f81c:40851 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;84486a41f81c:40851. 2024-12-04T10:09:15,102 DEBUG [RS:0;84486a41f81c:40851 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T10:09:15,103 DEBUG [RS:0;84486a41f81c:40851 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T10:09:15,103 INFO [RS:0;84486a41f81c:40851 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T10:09:15,103 INFO [RS:0;84486a41f81c:40851 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T10:09:15,103 INFO [RS:0;84486a41f81c:40851 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T10:09:15,104 INFO [RS:0;84486a41f81c:40851 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-04T10:09:15,104 INFO [RS:0;84486a41f81c:40851 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-04T10:09:15,104 DEBUG [RS:0;84486a41f81c:40851 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-04T10:09:15,104 DEBUG [RS:0;84486a41f81c:40851 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T10:09:15,104 INFO [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-04T10:09:15,105 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T10:09:15,105 INFO [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T10:09:15,105 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T10:09:15,105 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T10:09:15,105 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T10:09:15,106 INFO [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-12-04T10:09:15,118 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 371b9d2467974d658e06e75393049245 2024-12-04T10:09:15,128 INFO [regionserver/84486a41f81c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T10:09:15,129 INFO [regionserver/84486a41f81c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T10:09:15,135 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740/.tmp/info/371b9d2467974d658e06e75393049245 is 205, key is testBlockEvictionOnRegionMove1733306952661,,1733306952678.2a1a1c56ffa4e926628a5dd6cb93f7b9./info:regioninfo/1733306955000/Put/seqid=0 2024-12-04T10:09:15,136 INFO [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740/.tmp/info/371b9d2467974d658e06e75393049245 2024-12-04T10:09:15,144 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on def5faf25f17430da5e24089ca242870 2024-12-04T10:09:15,154 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740/.tmp/ns/def5faf25f17430da5e24089ca242870 is 43, key is default/ns:d/1733306952482/Put/seqid=0 2024-12-04T10:09:15,155 INFO [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740/.tmp/ns/def5faf25f17430da5e24089ca242870 2024-12-04T10:09:15,162 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 2c641ae315604d03b2a65d55b390be57 2024-12-04T10:09:15,174 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740/.tmp/table/2c641ae315604d03b2a65d55b390be57 is 78, key is testBlockEvictionOnRegionMove1733306952661/table:state/1733306953087/Put/seqid=0 2024-12-04T10:09:15,174 INFO [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740/.tmp/table/2c641ae315604d03b2a65d55b390be57 2024-12-04T10:09:15,176 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740/.tmp/info/371b9d2467974d658e06e75393049245 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740/info/371b9d2467974d658e06e75393049245 2024-12-04T10:09:15,178 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740/info/371b9d2467974d658e06e75393049245, delay=989 ms 2024-12-04T10:09:15,179 INFO [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740/info/371b9d2467974d658e06e75393049245, entries=15, sequenceid=15, filesize=7.6 K 2024-12-04T10:09:15,179 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740/.tmp/ns/def5faf25f17430da5e24089ca242870 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740/ns/def5faf25f17430da5e24089ca242870 2024-12-04T10:09:15,180 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740/ns/def5faf25f17430da5e24089ca242870, delay=923 ms 2024-12-04T10:09:15,181 INFO [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740/ns/def5faf25f17430da5e24089ca242870, entries=2, sequenceid=15, filesize=5.0 K 2024-12-04T10:09:15,182 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740/.tmp/table/2c641ae315604d03b2a65d55b390be57 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740/table/2c641ae315604d03b2a65d55b390be57 2024-12-04T10:09:15,183 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740/table/2c641ae315604d03b2a65d55b390be57, delay=978 ms 2024-12-04T10:09:15,184 INFO [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740/table/2c641ae315604d03b2a65d55b390be57, entries=2, sequenceid=15, filesize=5.3 K 2024-12-04T10:09:15,184 INFO [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 79ms, sequenceid=15, compaction requested=false 2024-12-04T10:09:15,185 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-04T10:09:15,186 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740/info/371b9d2467974d658e06e75393049245 2024-12-04T10:09:15,189 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740/ns/def5faf25f17430da5e24089ca242870 2024-12-04T10:09:15,190 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740/table/2c641ae315604d03b2a65d55b390be57 2024-12-04T10:09:15,195 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-12-04T10:09:15,196 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T10:09:15,196 INFO [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T10:09:15,196 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733306955104Running coprocessor pre-close hooks at 1733306955105 (+1 ms)Disabling compacts and flushes for region at 1733306955105Disabling writes for close at 1733306955105Obtaining lock to block concurrent updates at 1733306955106 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1733306955106Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1733306955106Flushing stores of hbase:meta,,1.1588230740 at 1733306955107 (+1 ms)Flushing 1588230740/info: creating writer at 1733306955107Flushing 1588230740/info: appending metadata at 1733306955134 (+27 ms)Flushing 1588230740/info: closing flushed file at 1733306955134Flushing 1588230740/ns: creating writer at 1733306955138 (+4 ms)Flushing 1588230740/ns: appending metadata at 1733306955154 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1733306955154Flushing 1588230740/table: creating writer at 1733306955157 (+3 ms)Flushing 1588230740/table: appending metadata at 1733306955173 (+16 ms)Flushing 1588230740/table: closing flushed file at 1733306955173Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@49430f86: reopening flushed file at 1733306955176 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@719ad481: reopening flushed file at 1733306955179 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@796aa0aa: reopening flushed file at 1733306955181 (+2 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 79ms, sequenceid=15, compaction requested=false at 1733306955184 (+3 ms)Writing region close event to WAL at 1733306955191 (+7 ms)Running coprocessor post-close hooks at 1733306955196 (+5 ms)Closed at 1733306955196 2024-12-04T10:09:15,197 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-04T10:09:15,267 INFO [RS:1;84486a41f81c:46729 {}] regionserver.HRegionServer(976): stopping server 84486a41f81c,46729,1733306950524; all regions closed. 2024-12-04T10:09:15,276 DEBUG [RS:1;84486a41f81c:46729 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/oldWALs 2024-12-04T10:09:15,276 INFO [RS:1;84486a41f81c:46729 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 84486a41f81c%2C46729%2C1733306950524:(num 1733306951883) 2024-12-04T10:09:15,277 DEBUG [RS:1;84486a41f81c:46729 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T10:09:15,277 INFO [RS:1;84486a41f81c:46729 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T10:09:15,277 INFO [RS:1;84486a41f81c:46729 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T10:09:15,277 INFO [RS:1;84486a41f81c:46729 {}] hbase.ChoreService(370): Chore service for: regionserver/84486a41f81c:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-04T10:09:15,278 INFO [RS:1;84486a41f81c:46729 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T10:09:15,278 INFO [RS:1;84486a41f81c:46729 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T10:09:15,278 INFO [regionserver/84486a41f81c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T10:09:15,278 INFO [RS:1;84486a41f81c:46729 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T10:09:15,278 INFO [RS:1;84486a41f81c:46729 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T10:09:15,279 INFO [RS:1;84486a41f81c:46729 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46729 2024-12-04T10:09:15,293 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46729-0x101a11ac1ab0002, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/84486a41f81c,46729,1733306950524 2024-12-04T10:09:15,293 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37539-0x101a11ac1ab0000, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T10:09:15,294 INFO [RS:1;84486a41f81c:46729 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T10:09:15,305 INFO [RS:0;84486a41f81c:40851 {}] regionserver.HRegionServer(976): stopping server 84486a41f81c,40851,1733306950404; all regions closed. 2024-12-04T10:09:15,305 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [84486a41f81c,46729,1733306950524] 2024-12-04T10:09:15,309 DEBUG [RS:0;84486a41f81c:40851 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/oldWALs 2024-12-04T10:09:15,309 INFO [RS:0;84486a41f81c:40851 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 84486a41f81c%2C40851%2C1733306950404.meta:.meta(num 1733306952362) 2024-12-04T10:09:15,312 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/84486a41f81c,46729,1733306950524 already deleted, retry=false 2024-12-04T10:09:15,313 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 84486a41f81c,46729,1733306950524 expired; onlineServers=1 2024-12-04T10:09:15,313 DEBUG [RS:0;84486a41f81c:40851 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/oldWALs 2024-12-04T10:09:15,313 INFO [RS:0;84486a41f81c:40851 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 84486a41f81c%2C40851%2C1733306950404:(num 1733306951883) 2024-12-04T10:09:15,313 DEBUG [RS:0;84486a41f81c:40851 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T10:09:15,314 INFO [RS:0;84486a41f81c:40851 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T10:09:15,314 INFO [RS:0;84486a41f81c:40851 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T10:09:15,314 INFO [RS:0;84486a41f81c:40851 {}] hbase.ChoreService(370): Chore service for: regionserver/84486a41f81c:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-04T10:09:15,314 INFO [RS:0;84486a41f81c:40851 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T10:09:15,314 INFO [regionserver/84486a41f81c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T10:09:15,315 INFO [RS:0;84486a41f81c:40851 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40851 2024-12-04T10:09:15,320 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40851-0x101a11ac1ab0001, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/84486a41f81c,40851,1733306950404 2024-12-04T10:09:15,320 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37539-0x101a11ac1ab0000, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T10:09:15,321 INFO [RS:0;84486a41f81c:40851 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T10:09:15,329 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [84486a41f81c,40851,1733306950404] 2024-12-04T10:09:15,337 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/84486a41f81c,40851,1733306950404 already deleted, retry=false 2024-12-04T10:09:15,337 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 84486a41f81c,40851,1733306950404 expired; onlineServers=0 2024-12-04T10:09:15,337 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '84486a41f81c,37539,1733306949675' ***** 2024-12-04T10:09:15,337 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-04T10:09:15,337 INFO [M:0;84486a41f81c:37539 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T10:09:15,338 INFO [M:0;84486a41f81c:37539 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T10:09:15,338 DEBUG [M:0;84486a41f81c:37539 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-04T10:09:15,338 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-04T10:09:15,338 DEBUG [M:0;84486a41f81c:37539 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-04T10:09:15,338 DEBUG [master/84486a41f81c:0:becomeActiveMaster-HFileCleaner.small.0-1733306951577 {}] cleaner.HFileCleaner(306): Exit Thread[master/84486a41f81c:0:becomeActiveMaster-HFileCleaner.small.0-1733306951577,5,FailOnTimeoutGroup] 2024-12-04T10:09:15,338 DEBUG [master/84486a41f81c:0:becomeActiveMaster-HFileCleaner.large.0-1733306951576 {}] cleaner.HFileCleaner(306): Exit Thread[master/84486a41f81c:0:becomeActiveMaster-HFileCleaner.large.0-1733306951576,5,FailOnTimeoutGroup] 2024-12-04T10:09:15,338 INFO [M:0;84486a41f81c:37539 {}] hbase.ChoreService(370): Chore service for: master/84486a41f81c:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-04T10:09:15,338 INFO [M:0;84486a41f81c:37539 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T10:09:15,339 DEBUG [M:0;84486a41f81c:37539 {}] master.HMaster(1795): Stopping service threads 2024-12-04T10:09:15,339 INFO [M:0;84486a41f81c:37539 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-04T10:09:15,339 INFO [M:0;84486a41f81c:37539 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T10:09:15,339 INFO [M:0;84486a41f81c:37539 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-04T10:09:15,340 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-04T10:09:15,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37539-0x101a11ac1ab0000, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-04T10:09:15,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37539-0x101a11ac1ab0000, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T10:09:15,346 DEBUG [M:0;84486a41f81c:37539 {}] zookeeper.ZKUtil(347): master:37539-0x101a11ac1ab0000, quorum=127.0.0.1:58820, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-04T10:09:15,346 WARN [M:0;84486a41f81c:37539 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-04T10:09:15,346 INFO [M:0;84486a41f81c:37539 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/.lastflushedseqids 2024-12-04T10:09:15,355 INFO [M:0;84486a41f81c:37539 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-04T10:09:15,356 INFO [M:0;84486a41f81c:37539 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-04T10:09:15,356 DEBUG [M:0;84486a41f81c:37539 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T10:09:15,356 INFO [M:0;84486a41f81c:37539 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T10:09:15,356 DEBUG [M:0;84486a41f81c:37539 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T10:09:15,356 DEBUG [M:0;84486a41f81c:37539 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T10:09:15,356 DEBUG [M:0;84486a41f81c:37539 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T10:09:15,357 INFO [M:0;84486a41f81c:37539 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.67 KB heapSize=43.02 KB 2024-12-04T10:09:15,365 DEBUG [M:0;84486a41f81c:37539 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on bb7d8d20c0504b8491defd3e5185ae0d 2024-12-04T10:09:15,376 DEBUG [M:0;84486a41f81c:37539 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bb7d8d20c0504b8491defd3e5185ae0d is 82, key is hbase:meta,,1/info:regioninfo/1733306952417/Put/seqid=0 2024-12-04T10:09:15,376 INFO [M:0;84486a41f81c:37539 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bb7d8d20c0504b8491defd3e5185ae0d 2024-12-04T10:09:15,383 DEBUG [M:0;84486a41f81c:37539 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 4fa16bb384cb4448b7df06ca09a0ea21 2024-12-04T10:09:15,394 DEBUG [M:0;84486a41f81c:37539 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4fa16bb384cb4448b7df06ca09a0ea21 is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733306953092/Put/seqid=0 2024-12-04T10:09:15,395 INFO [M:0;84486a41f81c:37539 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.05 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4fa16bb384cb4448b7df06ca09a0ea21 2024-12-04T10:09:15,401 DEBUG [M:0;84486a41f81c:37539 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d9086577ea8d40228d16f19276612c58 2024-12-04T10:09:15,405 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46729-0x101a11ac1ab0002, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T10:09:15,405 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46729-0x101a11ac1ab0002, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T10:09:15,405 INFO [RS:1;84486a41f81c:46729 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T10:09:15,405 INFO [RS:1;84486a41f81c:46729 {}] regionserver.HRegionServer(1031): Exiting; stopping=84486a41f81c,46729,1733306950524; zookeeper connection closed. 2024-12-04T10:09:15,406 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6519ac2f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6519ac2f 2024-12-04T10:09:15,412 DEBUG [M:0;84486a41f81c:37539 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d9086577ea8d40228d16f19276612c58 is 69, key is 84486a41f81c,40851,1733306950404/rs:state/1733306951618/Put/seqid=0 2024-12-04T10:09:15,413 INFO [M:0;84486a41f81c:37539 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d9086577ea8d40228d16f19276612c58 2024-12-04T10:09:15,415 DEBUG [M:0;84486a41f81c:37539 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bb7d8d20c0504b8491defd3e5185ae0d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/bb7d8d20c0504b8491defd3e5185ae0d 2024-12-04T10:09:15,416 INFO [M:0;84486a41f81c:37539 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/bb7d8d20c0504b8491defd3e5185ae0d, entries=8, sequenceid=84, filesize=5.5 K 2024-12-04T10:09:15,416 DEBUG [M:0;84486a41f81c:37539 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4fa16bb384cb4448b7df06ca09a0ea21 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4fa16bb384cb4448b7df06ca09a0ea21 2024-12-04T10:09:15,418 INFO [M:0;84486a41f81c:37539 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4fa16bb384cb4448b7df06ca09a0ea21, entries=9, sequenceid=84, filesize=6.5 K 2024-12-04T10:09:15,418 DEBUG [M:0;84486a41f81c:37539 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d9086577ea8d40228d16f19276612c58 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d9086577ea8d40228d16f19276612c58 2024-12-04T10:09:15,420 INFO [M:0;84486a41f81c:37539 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d9086577ea8d40228d16f19276612c58, entries=2, sequenceid=84, filesize=5.1 K 2024-12-04T10:09:15,420 INFO [M:0;84486a41f81c:37539 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 64ms, sequenceid=84, compaction requested=false 2024-12-04T10:09:15,422 INFO [M:0;84486a41f81c:37539 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T10:09:15,422 DEBUG [M:0;84486a41f81c:37539 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733306955356Disabling compacts and flushes for region at 1733306955356Disabling writes for close at 1733306955356Obtaining lock to block concurrent updates at 1733306955357 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733306955357Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35506, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1733306955357Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733306955358 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733306955358Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733306955375 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733306955375Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733306955378 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733306955394 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733306955394Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733306955396 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733306955412 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733306955412Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@55dd33e8: reopening flushed file at 1733306955414 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@798891a2: reopening flushed file at 1733306955416 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7472a485: reopening flushed file at 1733306955418 (+2 ms)Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 64ms, sequenceid=84, compaction requested=false at 1733306955420 (+2 ms)Writing region close event to WAL at 1733306955422 (+2 ms)Closed at 1733306955422 2024-12-04T10:09:15,424 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T10:09:15,424 INFO [M:0;84486a41f81c:37539 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-04T10:09:15,424 INFO [M:0;84486a41f81c:37539 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37539 2024-12-04T10:09:15,424 INFO [M:0;84486a41f81c:37539 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T10:09:15,429 INFO [RS:0;84486a41f81c:40851 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T10:09:15,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40851-0x101a11ac1ab0001, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T10:09:15,429 INFO [RS:0;84486a41f81c:40851 {}] regionserver.HRegionServer(1031): Exiting; stopping=84486a41f81c,40851,1733306950404; zookeeper connection closed. 2024-12-04T10:09:15,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40851-0x101a11ac1ab0001, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T10:09:15,429 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@768a8158 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@768a8158 2024-12-04T10:09:15,430 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-12-04T10:09:15,529 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37539-0x101a11ac1ab0000, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T10:09:15,529 INFO [M:0;84486a41f81c:37539 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T10:09:15,529 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37539-0x101a11ac1ab0000, quorum=127.0.0.1:58820, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T10:09:15,537 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-04T10:09:15,556 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-04T10:09:15,557 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T10:09:15,564 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=50 (was 12) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733306953444 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: hfile-prefetch-1733306955180 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/84486a41f81c:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: hfile-prefetch-1733306954984 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/84486a41f81c:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: hfile-prefetch-1733306955178 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/84486a41f81c:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/84486a41f81c:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=830 (was 850), ProcessCount=11 (was 11), AvailableMemoryMB=10639 (was 10978) 2024-12-04T10:09:15,569 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=50, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=830, ProcessCount=11, AvailableMemoryMB=10639 2024-12-04T10:09:15,569 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/hadoop-log-dir so I do NOT create it in target/test-data/d829fef9-ca4a-35f8-a2bc-ee23e1db7a16 2024-12-04T10:09:15,569 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/hadoop-tmp-dir so I do NOT create it in target/test-data/d829fef9-ca4a-35f8-a2bc-ee23e1db7a16 2024-12-04T10:09:15,569 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T10:09:15,572 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d829fef9-ca4a-35f8-a2bc-ee23e1db7a16/cluster_c6a2a512-8c28-5dad-59cd-63dedb494164, deleteOnExit=true 2024-12-04T10:09:15,588 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d829fef9-ca4a-35f8-a2bc-ee23e1db7a16/cluster_c6a2a512-8c28-5dad-59cd-63dedb494164/zookeeper_0, clientPort=57330, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d829fef9-ca4a-35f8-a2bc-ee23e1db7a16/cluster_c6a2a512-8c28-5dad-59cd-63dedb494164/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d829fef9-ca4a-35f8-a2bc-ee23e1db7a16/cluster_c6a2a512-8c28-5dad-59cd-63dedb494164/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-04T10:09:15,589 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57330 2024-12-04T10:09:15,603 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee with version=8 2024-12-04T10:09:15,603 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/hbase-staging 2024-12-04T10:09:15,607 INFO [Time-limited test {}] client.ConnectionUtils(128): master/84486a41f81c:0 server-side Connection retries=45 2024-12-04T10:09:15,607 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T10:09:15,607 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T10:09:15,607 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T10:09:15,607 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T10:09:15,607 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T10:09:15,607 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-04T10:09:15,608 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T10:09:15,608 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37329 2024-12-04T10:09:15,610 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37329 connecting to ZooKeeper ensemble=127.0.0.1:57330 2024-12-04T10:09:15,659 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:373290x0, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T10:09:15,660 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37329-0x101a11ad9b80000 connected 2024-12-04T10:09:15,744 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T10:09:15,745 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T10:09:15,746 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37329-0x101a11ad9b80000, quorum=127.0.0.1:57330, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T10:09:15,746 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee, hbase.cluster.distributed=false 2024-12-04T10:09:15,750 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37329-0x101a11ad9b80000, quorum=127.0.0.1:57330, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T10:09:15,751 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37329 2024-12-04T10:09:15,751 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37329 2024-12-04T10:09:15,752 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37329 2024-12-04T10:09:15,753 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37329 2024-12-04T10:09:15,753 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37329 2024-12-04T10:09:15,770 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/84486a41f81c:0 server-side Connection retries=45 2024-12-04T10:09:15,770 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T10:09:15,770 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T10:09:15,770 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T10:09:15,770 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T10:09:15,770 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T10:09:15,770 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T10:09:15,770 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T10:09:15,771 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46309 2024-12-04T10:09:15,772 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46309 connecting to ZooKeeper ensemble=127.0.0.1:57330 2024-12-04T10:09:15,772 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T10:09:15,772 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T10:09:15,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:463090x0, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T10:09:15,780 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46309-0x101a11ad9b80001 connected 2024-12-04T10:09:15,780 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46309-0x101a11ad9b80001, quorum=127.0.0.1:57330, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T10:09:15,781 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T10:09:15,781 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d829fef9-ca4a-35f8-a2bc-ee23e1db7a16/bucket.cache 2024-12-04T10:09:15,781 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-04T10:09:15,781 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-04T10:09:15,781 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-04T10:09:15,782 INFO [Thread-118 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-04T10:09:15,782 WARN [Thread-118 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-04T10:09:15,782 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d829fef9-ca4a-35f8-a2bc-ee23e1db7a16/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d829fef9-ca4a-35f8-a2bc-ee23e1db7a16/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-04T10:09:15,782 INFO [Thread-118 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-04T10:09:15,782 INFO [Thread-118 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d829fef9-ca4a-35f8-a2bc-ee23e1db7a16/bucket.persistence is complete. 2024-12-04T10:09:15,782 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T10:09:15,783 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46309-0x101a11ad9b80001, quorum=127.0.0.1:57330, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T10:09:15,784 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46309-0x101a11ad9b80001, quorum=127.0.0.1:57330, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T10:09:15,786 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46309 2024-12-04T10:09:15,786 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46309 2024-12-04T10:09:15,786 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46309 2024-12-04T10:09:15,787 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46309 2024-12-04T10:09:15,787 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46309 2024-12-04T10:09:15,803 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/84486a41f81c:0 server-side Connection retries=45 2024-12-04T10:09:15,803 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T10:09:15,803 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T10:09:15,803 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T10:09:15,803 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T10:09:15,803 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T10:09:15,803 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T10:09:15,803 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T10:09:15,804 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45753 2024-12-04T10:09:15,805 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45753 connecting to ZooKeeper ensemble=127.0.0.1:57330 2024-12-04T10:09:15,805 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T10:09:15,805 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T10:09:15,813 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:457530x0, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T10:09:15,813 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45753-0x101a11ad9b80002 connected 2024-12-04T10:09:15,813 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45753-0x101a11ad9b80002, quorum=127.0.0.1:57330, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T10:09:15,814 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T10:09:15,814 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d829fef9-ca4a-35f8-a2bc-ee23e1db7a16/bucket.cache 2024-12-04T10:09:15,814 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-04T10:09:15,814 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-04T10:09:15,814 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-04T10:09:15,815 INFO [Thread-119 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-04T10:09:15,815 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d829fef9-ca4a-35f8-a2bc-ee23e1db7a16/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d829fef9-ca4a-35f8-a2bc-ee23e1db7a16/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-04T10:09:15,815 WARN [Thread-119 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-04T10:09:15,815 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T10:09:15,815 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-04T10:09:15,815 INFO [Thread-119 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d829fef9-ca4a-35f8-a2bc-ee23e1db7a16/bucket.persistence is complete. 2024-12-04T10:09:15,816 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45753-0x101a11ad9b80002, quorum=127.0.0.1:57330, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T10:09:15,817 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45753-0x101a11ad9b80002, quorum=127.0.0.1:57330, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T10:09:15,819 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45753 2024-12-04T10:09:15,819 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45753 2024-12-04T10:09:15,819 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45753 2024-12-04T10:09:15,819 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45753 2024-12-04T10:09:15,820 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45753 2024-12-04T10:09:15,830 DEBUG [M:0;84486a41f81c:37329 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;84486a41f81c:37329 2024-12-04T10:09:15,831 INFO [master/84486a41f81c:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/84486a41f81c,37329,1733306955606 2024-12-04T10:09:15,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46309-0x101a11ad9b80001, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T10:09:15,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45753-0x101a11ad9b80002, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T10:09:15,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37329-0x101a11ad9b80000, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T10:09:15,838 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37329-0x101a11ad9b80000, quorum=127.0.0.1:57330, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/84486a41f81c,37329,1733306955606 2024-12-04T10:09:15,846 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37329-0x101a11ad9b80000, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T10:09:15,846 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46309-0x101a11ad9b80001, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T10:09:15,846 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45753-0x101a11ad9b80002, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T10:09:15,846 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46309-0x101a11ad9b80001, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T10:09:15,846 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45753-0x101a11ad9b80002, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T10:09:15,847 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37329-0x101a11ad9b80000, quorum=127.0.0.1:57330, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-04T10:09:15,847 INFO [master/84486a41f81c:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/84486a41f81c,37329,1733306955606 from backup master directory 2024-12-04T10:09:15,854 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46309-0x101a11ad9b80001, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T10:09:15,854 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45753-0x101a11ad9b80002, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T10:09:15,854 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37329-0x101a11ad9b80000, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/84486a41f81c,37329,1733306955606 2024-12-04T10:09:15,854 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37329-0x101a11ad9b80000, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T10:09:15,854 WARN [master/84486a41f81c:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T10:09:15,855 INFO [master/84486a41f81c:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=84486a41f81c,37329,1733306955606 2024-12-04T10:09:15,856 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/hbase.id] with ID: 0117e7a0-3c2e-4ff9-bc50-889e5f40698d 2024-12-04T10:09:15,856 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/.tmp/hbase.id 2024-12-04T10:09:15,858 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/hbase.id] 2024-12-04T10:09:15,886 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T10:09:15,886 INFO [master/84486a41f81c:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-04T10:09:15,888 INFO [master/84486a41f81c:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-12-04T10:09:15,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45753-0x101a11ad9b80002, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T10:09:15,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37329-0x101a11ad9b80000, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T10:09:15,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46309-0x101a11ad9b80001, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T10:09:15,901 INFO [master/84486a41f81c:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T10:09:15,903 INFO [master/84486a41f81c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-04T10:09:15,903 INFO [master/84486a41f81c:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T10:09:15,926 INFO [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/MasterData/data/master/store 2024-12-04T10:09:15,932 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T10:09:15,932 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T10:09:15,933 INFO [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T10:09:15,933 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T10:09:15,933 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T10:09:15,933 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T10:09:15,933 INFO [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T10:09:15,933 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733306955932Disabling compacts and flushes for region at 1733306955932Disabling writes for close at 1733306955933 (+1 ms)Writing region close event to WAL at 1733306955933Closed at 1733306955933 2024-12-04T10:09:15,933 WARN [master/84486a41f81c:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/MasterData/data/master/store/.initializing 2024-12-04T10:09:15,933 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/MasterData/WALs/84486a41f81c,37329,1733306955606 2024-12-04T10:09:15,940 INFO [master/84486a41f81c:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T10:09:15,944 INFO [master/84486a41f81c:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=84486a41f81c%2C37329%2C1733306955606, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/MasterData/WALs/84486a41f81c,37329,1733306955606, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/MasterData/oldWALs, maxLogs=10 2024-12-04T10:09:15,961 INFO [master/84486a41f81c:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/MasterData/WALs/84486a41f81c,37329,1733306955606/84486a41f81c%2C37329%2C1733306955606.1733306955944 2024-12-04T10:09:15,961 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-04T10:09:15,961 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-04T10:09:15,961 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T10:09:15,961 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T10:09:15,961 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T10:09:15,962 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T10:09:15,966 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-04T10:09:15,966 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T10:09:15,967 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T10:09:15,967 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T10:09:15,970 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-04T10:09:15,970 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T10:09:15,971 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T10:09:15,971 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T10:09:15,974 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-04T10:09:15,974 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T10:09:15,974 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T10:09:15,975 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T10:09:15,978 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-04T10:09:15,978 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T10:09:15,978 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T10:09:15,979 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T10:09:15,979 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T10:09:15,979 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T10:09:15,979 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T10:09:15,979 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T10:09:15,980 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T10:09:15,980 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T10:09:15,985 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T10:09:15,986 INFO [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60294991, jitterRate=-0.10153461992740631}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T10:09:15,987 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733306955962Initializing all the Stores at 1733306955962Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733306955962Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733306955962Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733306955962Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733306955962Cleaning up temporary data from old regions at 1733306955979 (+17 ms)Region opened successfully at 1733306955986 (+7 ms) 2024-12-04T10:09:15,987 INFO [master/84486a41f81c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-04T10:09:15,991 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dc656f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=84486a41f81c/172.17.0.2:0 2024-12-04T10:09:15,992 INFO [master/84486a41f81c:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-04T10:09:15,992 INFO [master/84486a41f81c:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-04T10:09:15,992 INFO [master/84486a41f81c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-04T10:09:15,992 INFO [master/84486a41f81c:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-04T10:09:15,992 INFO [master/84486a41f81c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-04T10:09:15,993 INFO [master/84486a41f81c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-12-04T10:09:15,993 INFO [master/84486a41f81c:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-04T10:09:15,995 INFO [master/84486a41f81c:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-04T10:09:15,996 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37329-0x101a11ad9b80000, quorum=127.0.0.1:57330, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-04T10:09:16,004 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-04T10:09:16,005 INFO [master/84486a41f81c:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-04T10:09:16,005 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37329-0x101a11ad9b80000, quorum=127.0.0.1:57330, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-04T10:09:16,013 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-04T10:09:16,013 INFO [master/84486a41f81c:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-04T10:09:16,014 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37329-0x101a11ad9b80000, quorum=127.0.0.1:57330, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-04T10:09:16,021 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-04T10:09:16,022 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37329-0x101a11ad9b80000, quorum=127.0.0.1:57330, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-04T10:09:16,029 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-04T10:09:16,032 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37329-0x101a11ad9b80000, quorum=127.0.0.1:57330, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-04T10:09:16,038 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-04T10:09:16,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45753-0x101a11ad9b80002, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T10:09:16,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37329-0x101a11ad9b80000, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T10:09:16,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46309-0x101a11ad9b80001, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T10:09:16,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45753-0x101a11ad9b80002, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T10:09:16,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37329-0x101a11ad9b80000, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T10:09:16,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46309-0x101a11ad9b80001, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T10:09:16,047 INFO [master/84486a41f81c:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=84486a41f81c,37329,1733306955606, sessionid=0x101a11ad9b80000, setting cluster-up flag (Was=false) 2024-12-04T10:09:16,063 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46309-0x101a11ad9b80001, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T10:09:16,063 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45753-0x101a11ad9b80002, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T10:09:16,063 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37329-0x101a11ad9b80000, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T10:09:16,088 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-04T10:09:16,090 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=84486a41f81c,37329,1733306955606 2024-12-04T10:09:16,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45753-0x101a11ad9b80002, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T10:09:16,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37329-0x101a11ad9b80000, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T10:09:16,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46309-0x101a11ad9b80001, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T10:09:16,129 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-04T10:09:16,131 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=84486a41f81c,37329,1733306955606 2024-12-04T10:09:16,131 INFO [master/84486a41f81c:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-04T10:09:16,133 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-04T10:09:16,133 INFO [master/84486a41f81c:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-04T10:09:16,133 INFO [master/84486a41f81c:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-04T10:09:16,133 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 84486a41f81c,37329,1733306955606 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-04T10:09:16,134 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/84486a41f81c:0, corePoolSize=5, maxPoolSize=5 2024-12-04T10:09:16,135 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/84486a41f81c:0, corePoolSize=5, maxPoolSize=5 2024-12-04T10:09:16,135 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/84486a41f81c:0, corePoolSize=5, maxPoolSize=5 2024-12-04T10:09:16,135 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/84486a41f81c:0, corePoolSize=5, maxPoolSize=5 2024-12-04T10:09:16,135 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/84486a41f81c:0, corePoolSize=10, maxPoolSize=10 2024-12-04T10:09:16,135 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:16,135 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/84486a41f81c:0, corePoolSize=2, maxPoolSize=2 2024-12-04T10:09:16,135 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:16,136 INFO [master/84486a41f81c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733306986136 2024-12-04T10:09:16,137 INFO [master/84486a41f81c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-04T10:09:16,137 INFO [master/84486a41f81c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-04T10:09:16,137 INFO [master/84486a41f81c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-04T10:09:16,137 INFO [master/84486a41f81c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-04T10:09:16,137 INFO [master/84486a41f81c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-04T10:09:16,137 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T10:09:16,137 INFO [master/84486a41f81c:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-04T10:09:16,137 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-04T10:09:16,137 INFO [master/84486a41f81c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:16,138 INFO [master/84486a41f81c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-04T10:09:16,138 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T10:09:16,138 INFO [master/84486a41f81c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-04T10:09:16,138 INFO [master/84486a41f81c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-04T10:09:16,138 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-04T10:09:16,138 INFO [master/84486a41f81c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-04T10:09:16,138 INFO [master/84486a41f81c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-04T10:09:16,139 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/84486a41f81c:0:becomeActiveMaster-HFileCleaner.large.0-1733306956139,5,FailOnTimeoutGroup] 2024-12-04T10:09:16,140 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/84486a41f81c:0:becomeActiveMaster-HFileCleaner.small.0-1733306956140,5,FailOnTimeoutGroup] 2024-12-04T10:09:16,140 INFO [master/84486a41f81c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:16,140 INFO [master/84486a41f81c:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-04T10:09:16,140 INFO [master/84486a41f81c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:16,140 INFO [master/84486a41f81c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:16,151 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-04T10:09:16,151 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee 2024-12-04T10:09:16,156 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T10:09:16,157 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-04T10:09:16,159 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T10:09:16,159 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T10:09:16,160 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T10:09:16,160 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-04T10:09:16,162 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T10:09:16,163 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T10:09:16,163 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T10:09:16,163 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-04T10:09:16,165 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T10:09:16,165 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T10:09:16,166 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T10:09:16,166 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-04T10:09:16,169 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T10:09:16,169 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T10:09:16,169 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T10:09:16,169 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T10:09:16,170 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740 2024-12-04T10:09:16,170 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740 2024-12-04T10:09:16,171 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T10:09:16,171 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T10:09:16,171 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T10:09:16,172 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T10:09:16,176 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T10:09:16,177 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72285964, jitterRate=0.0771448016166687}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T10:09:16,177 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733306956156Initializing all the Stores at 1733306956156Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733306956156Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733306956156Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733306956156Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733306956156Cleaning up temporary data from old regions at 1733306956171 (+15 ms)Region opened successfully at 1733306956177 (+6 ms) 2024-12-04T10:09:16,177 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T10:09:16,177 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T10:09:16,177 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T10:09:16,178 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T10:09:16,178 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T10:09:16,178 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T10:09:16,178 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733306956177Disabling compacts and flushes for region at 1733306956177Disabling writes for close at 1733306956178 (+1 ms)Writing region close event to WAL at 1733306956178Closed at 1733306956178 2024-12-04T10:09:16,179 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T10:09:16,179 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-04T10:09:16,179 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-04T10:09:16,180 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T10:09:16,181 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-04T10:09:16,224 INFO [RS:1;84486a41f81c:45753 {}] regionserver.HRegionServer(746): ClusterId : 0117e7a0-3c2e-4ff9-bc50-889e5f40698d 2024-12-04T10:09:16,224 INFO [RS:0;84486a41f81c:46309 {}] regionserver.HRegionServer(746): ClusterId : 0117e7a0-3c2e-4ff9-bc50-889e5f40698d 2024-12-04T10:09:16,224 DEBUG [RS:1;84486a41f81c:45753 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T10:09:16,224 DEBUG [RS:0;84486a41f81c:46309 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T10:09:16,237 DEBUG [RS:0;84486a41f81c:46309 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T10:09:16,237 DEBUG [RS:1;84486a41f81c:45753 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T10:09:16,237 DEBUG [RS:0;84486a41f81c:46309 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T10:09:16,237 DEBUG [RS:1;84486a41f81c:45753 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T10:09:16,247 DEBUG [RS:1;84486a41f81c:45753 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T10:09:16,247 DEBUG [RS:0;84486a41f81c:46309 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T10:09:16,248 DEBUG [RS:0;84486a41f81c:46309 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6034e2b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=84486a41f81c/172.17.0.2:0 2024-12-04T10:09:16,248 DEBUG [RS:1;84486a41f81c:45753 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3694665, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=84486a41f81c/172.17.0.2:0 2024-12-04T10:09:16,260 DEBUG [RS:0;84486a41f81c:46309 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;84486a41f81c:46309 2024-12-04T10:09:16,260 DEBUG [RS:1;84486a41f81c:45753 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;84486a41f81c:45753 2024-12-04T10:09:16,260 INFO [RS:0;84486a41f81c:46309 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-04T10:09:16,260 INFO [RS:1;84486a41f81c:45753 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-04T10:09:16,260 INFO [RS:0;84486a41f81c:46309 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-04T10:09:16,260 INFO [RS:1;84486a41f81c:45753 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-04T10:09:16,260 DEBUG [RS:0;84486a41f81c:46309 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-04T10:09:16,260 DEBUG [RS:1;84486a41f81c:45753 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-04T10:09:16,261 INFO [RS:1;84486a41f81c:45753 {}] regionserver.HRegionServer(2659): reportForDuty to master=84486a41f81c,37329,1733306955606 with port=45753, startcode=1733306955802 2024-12-04T10:09:16,261 INFO [RS:0;84486a41f81c:46309 {}] regionserver.HRegionServer(2659): reportForDuty to master=84486a41f81c,37329,1733306955606 with port=46309, startcode=1733306955769 2024-12-04T10:09:16,261 DEBUG [RS:1;84486a41f81c:45753 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T10:09:16,261 DEBUG [RS:0;84486a41f81c:46309 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T10:09:16,264 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43261, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T10:09:16,264 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39951, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T10:09:16,265 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37329 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 84486a41f81c,45753,1733306955802 2024-12-04T10:09:16,265 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37329 {}] master.ServerManager(517): Registering regionserver=84486a41f81c,45753,1733306955802 2024-12-04T10:09:16,266 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37329 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 84486a41f81c,46309,1733306955769 2024-12-04T10:09:16,266 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37329 {}] master.ServerManager(517): Registering regionserver=84486a41f81c,46309,1733306955769 2024-12-04T10:09:16,266 DEBUG [RS:1;84486a41f81c:45753 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee 2024-12-04T10:09:16,266 DEBUG [RS:1;84486a41f81c:45753 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-04T10:09:16,266 DEBUG [RS:1;84486a41f81c:45753 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-04T10:09:16,267 DEBUG [RS:0;84486a41f81c:46309 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee 2024-12-04T10:09:16,267 DEBUG [RS:0;84486a41f81c:46309 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-04T10:09:16,267 DEBUG [RS:0;84486a41f81c:46309 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-04T10:09:16,277 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37329-0x101a11ad9b80000, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T10:09:16,309 DEBUG [RS:1;84486a41f81c:45753 {}] zookeeper.ZKUtil(111): regionserver:45753-0x101a11ad9b80002, quorum=127.0.0.1:57330, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/84486a41f81c,45753,1733306955802 2024-12-04T10:09:16,310 WARN [RS:1;84486a41f81c:45753 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T10:09:16,310 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [84486a41f81c,45753,1733306955802] 2024-12-04T10:09:16,310 INFO [RS:1;84486a41f81c:45753 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T10:09:16,310 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [84486a41f81c,46309,1733306955769] 2024-12-04T10:09:16,310 DEBUG [RS:1;84486a41f81c:45753 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/WALs/84486a41f81c,45753,1733306955802 2024-12-04T10:09:16,310 DEBUG [RS:0;84486a41f81c:46309 {}] zookeeper.ZKUtil(111): regionserver:46309-0x101a11ad9b80001, quorum=127.0.0.1:57330, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/84486a41f81c,46309,1733306955769 2024-12-04T10:09:16,311 WARN [RS:0;84486a41f81c:46309 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T10:09:16,311 INFO [RS:0;84486a41f81c:46309 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T10:09:16,311 DEBUG [RS:0;84486a41f81c:46309 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/WALs/84486a41f81c,46309,1733306955769 2024-12-04T10:09:16,319 INFO [RS:0;84486a41f81c:46309 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T10:09:16,319 INFO [RS:1;84486a41f81c:45753 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T10:09:16,322 INFO [RS:0;84486a41f81c:46309 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T10:09:16,322 INFO [RS:0;84486a41f81c:46309 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T10:09:16,322 INFO [RS:0;84486a41f81c:46309 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:16,322 INFO [RS:0;84486a41f81c:46309 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-04T10:09:16,323 INFO [RS:0;84486a41f81c:46309 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-04T10:09:16,323 INFO [RS:1;84486a41f81c:45753 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T10:09:16,323 INFO [RS:0;84486a41f81c:46309 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:16,323 DEBUG [RS:0;84486a41f81c:46309 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:16,323 DEBUG [RS:0;84486a41f81c:46309 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:16,323 DEBUG [RS:0;84486a41f81c:46309 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:16,323 DEBUG [RS:0;84486a41f81c:46309 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:16,323 INFO [RS:1;84486a41f81c:45753 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T10:09:16,323 DEBUG [RS:0;84486a41f81c:46309 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:16,323 INFO [RS:1;84486a41f81c:45753 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:16,324 DEBUG [RS:0;84486a41f81c:46309 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/84486a41f81c:0, corePoolSize=2, maxPoolSize=2 2024-12-04T10:09:16,324 DEBUG [RS:0;84486a41f81c:46309 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:16,324 INFO [RS:1;84486a41f81c:45753 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-04T10:09:16,324 DEBUG [RS:0;84486a41f81c:46309 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:16,324 DEBUG [RS:0;84486a41f81c:46309 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:16,324 DEBUG [RS:0;84486a41f81c:46309 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:16,324 DEBUG [RS:0;84486a41f81c:46309 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:16,324 INFO [RS:1;84486a41f81c:45753 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-04T10:09:16,324 DEBUG [RS:0;84486a41f81c:46309 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:16,324 DEBUG [RS:0;84486a41f81c:46309 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/84486a41f81c:0, corePoolSize=3, maxPoolSize=3 2024-12-04T10:09:16,324 INFO [RS:1;84486a41f81c:45753 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:16,324 DEBUG [RS:0;84486a41f81c:46309 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/84486a41f81c:0, corePoolSize=3, maxPoolSize=3 2024-12-04T10:09:16,325 DEBUG [RS:1;84486a41f81c:45753 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:16,325 DEBUG [RS:1;84486a41f81c:45753 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:16,325 DEBUG [RS:1;84486a41f81c:45753 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:16,325 DEBUG [RS:1;84486a41f81c:45753 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:16,325 DEBUG [RS:1;84486a41f81c:45753 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:16,325 DEBUG [RS:1;84486a41f81c:45753 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/84486a41f81c:0, corePoolSize=2, maxPoolSize=2 2024-12-04T10:09:16,325 DEBUG [RS:1;84486a41f81c:45753 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:16,325 DEBUG [RS:1;84486a41f81c:45753 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:16,325 INFO [RS:0;84486a41f81c:46309 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:16,325 DEBUG [RS:1;84486a41f81c:45753 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:16,325 INFO [RS:0;84486a41f81c:46309 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:16,325 INFO [RS:0;84486a41f81c:46309 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:16,325 DEBUG [RS:1;84486a41f81c:45753 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:16,325 INFO [RS:0;84486a41f81c:46309 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:16,325 DEBUG [RS:1;84486a41f81c:45753 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:16,325 INFO [RS:0;84486a41f81c:46309 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:16,325 INFO [RS:0;84486a41f81c:46309 {}] hbase.ChoreService(168): Chore ScheduledChore name=84486a41f81c,46309,1733306955769-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T10:09:16,325 DEBUG [RS:1;84486a41f81c:45753 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:16,326 DEBUG [RS:1;84486a41f81c:45753 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/84486a41f81c:0, corePoolSize=3, maxPoolSize=3 2024-12-04T10:09:16,326 DEBUG [RS:1;84486a41f81c:45753 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/84486a41f81c:0, corePoolSize=3, maxPoolSize=3 2024-12-04T10:09:16,328 INFO [RS:1;84486a41f81c:45753 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:16,328 INFO [RS:1;84486a41f81c:45753 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:16,328 INFO [RS:1;84486a41f81c:45753 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:16,328 INFO [RS:1;84486a41f81c:45753 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:16,328 INFO [RS:1;84486a41f81c:45753 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:16,328 INFO [RS:1;84486a41f81c:45753 {}] hbase.ChoreService(168): Chore ScheduledChore name=84486a41f81c,45753,1733306955802-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T10:09:16,331 WARN [84486a41f81c:37329 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T10:09:16,338 INFO [RS:0;84486a41f81c:46309 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T10:09:16,338 INFO [RS:0;84486a41f81c:46309 {}] hbase.ChoreService(168): Chore ScheduledChore name=84486a41f81c,46309,1733306955769-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:16,338 INFO [RS:0;84486a41f81c:46309 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:16,338 INFO [RS:0;84486a41f81c:46309 {}] regionserver.Replication(171): 84486a41f81c,46309,1733306955769 started 2024-12-04T10:09:16,340 INFO [RS:1;84486a41f81c:45753 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T10:09:16,340 INFO [RS:1;84486a41f81c:45753 {}] hbase.ChoreService(168): Chore ScheduledChore name=84486a41f81c,45753,1733306955802-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:16,340 INFO [RS:1;84486a41f81c:45753 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:16,340 INFO [RS:1;84486a41f81c:45753 {}] regionserver.Replication(171): 84486a41f81c,45753,1733306955802 started 2024-12-04T10:09:16,350 INFO [RS:0;84486a41f81c:46309 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:16,350 INFO [RS:0;84486a41f81c:46309 {}] regionserver.HRegionServer(1482): Serving as 84486a41f81c,46309,1733306955769, RpcServer on 84486a41f81c/172.17.0.2:46309, sessionid=0x101a11ad9b80001 2024-12-04T10:09:16,350 DEBUG [RS:0;84486a41f81c:46309 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T10:09:16,350 DEBUG [RS:0;84486a41f81c:46309 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 84486a41f81c,46309,1733306955769 2024-12-04T10:09:16,350 DEBUG [RS:0;84486a41f81c:46309 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '84486a41f81c,46309,1733306955769' 2024-12-04T10:09:16,350 DEBUG [RS:0;84486a41f81c:46309 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T10:09:16,351 DEBUG [RS:0;84486a41f81c:46309 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T10:09:16,351 INFO [RS:1;84486a41f81c:45753 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:16,351 INFO [RS:1;84486a41f81c:45753 {}] regionserver.HRegionServer(1482): Serving as 84486a41f81c,45753,1733306955802, RpcServer on 84486a41f81c/172.17.0.2:45753, sessionid=0x101a11ad9b80002 2024-12-04T10:09:16,351 DEBUG [RS:0;84486a41f81c:46309 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T10:09:16,351 DEBUG [RS:0;84486a41f81c:46309 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T10:09:16,351 DEBUG [RS:0;84486a41f81c:46309 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 84486a41f81c,46309,1733306955769 2024-12-04T10:09:16,352 DEBUG [RS:0;84486a41f81c:46309 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '84486a41f81c,46309,1733306955769' 2024-12-04T10:09:16,352 DEBUG [RS:1;84486a41f81c:45753 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T10:09:16,352 DEBUG [RS:0;84486a41f81c:46309 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T10:09:16,352 DEBUG [RS:1;84486a41f81c:45753 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 84486a41f81c,45753,1733306955802 2024-12-04T10:09:16,352 DEBUG [RS:1;84486a41f81c:45753 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '84486a41f81c,45753,1733306955802' 2024-12-04T10:09:16,352 DEBUG [RS:1;84486a41f81c:45753 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T10:09:16,352 DEBUG [RS:0;84486a41f81c:46309 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T10:09:16,352 DEBUG [RS:1;84486a41f81c:45753 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T10:09:16,353 DEBUG [RS:0;84486a41f81c:46309 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T10:09:16,353 DEBUG [RS:1;84486a41f81c:45753 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T10:09:16,353 INFO [RS:0;84486a41f81c:46309 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T10:09:16,353 DEBUG [RS:1;84486a41f81c:45753 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T10:09:16,353 INFO [RS:0;84486a41f81c:46309 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T10:09:16,353 DEBUG [RS:1;84486a41f81c:45753 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 84486a41f81c,45753,1733306955802 2024-12-04T10:09:16,353 DEBUG [RS:1;84486a41f81c:45753 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '84486a41f81c,45753,1733306955802' 2024-12-04T10:09:16,353 DEBUG [RS:1;84486a41f81c:45753 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T10:09:16,353 DEBUG [RS:1;84486a41f81c:45753 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T10:09:16,353 DEBUG [RS:1;84486a41f81c:45753 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T10:09:16,353 INFO [RS:1;84486a41f81c:45753 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T10:09:16,353 INFO [RS:1;84486a41f81c:45753 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T10:09:16,454 INFO [RS:0;84486a41f81c:46309 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T10:09:16,454 INFO [RS:1;84486a41f81c:45753 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T10:09:16,455 INFO [RS:0;84486a41f81c:46309 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=84486a41f81c%2C46309%2C1733306955769, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/WALs/84486a41f81c,46309,1733306955769, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/oldWALs, maxLogs=55 2024-12-04T10:09:16,456 INFO [RS:1;84486a41f81c:45753 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=84486a41f81c%2C45753%2C1733306955802, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/WALs/84486a41f81c,45753,1733306955802, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/oldWALs, maxLogs=55 2024-12-04T10:09:16,476 INFO [RS:0;84486a41f81c:46309 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/WALs/84486a41f81c,46309,1733306955769/84486a41f81c%2C46309%2C1733306955769.1733306956459 2024-12-04T10:09:16,476 INFO [RS:1;84486a41f81c:45753 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/WALs/84486a41f81c,45753,1733306955802/84486a41f81c%2C45753%2C1733306955802.1733306956460 2024-12-04T10:09:16,476 DEBUG [RS:0;84486a41f81c:46309 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-04T10:09:16,476 DEBUG [RS:1;84486a41f81c:45753 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-04T10:09:16,581 DEBUG [84486a41f81c:37329 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-04T10:09:16,583 DEBUG [84486a41f81c:37329 {}] balancer.BalancerClusterState(204): Hosts are {84486a41f81c=0} racks are {/default-rack=0} 2024-12-04T10:09:16,585 DEBUG [84486a41f81c:37329 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-04T10:09:16,585 DEBUG [84486a41f81c:37329 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-04T10:09:16,585 DEBUG [84486a41f81c:37329 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-04T10:09:16,585 DEBUG [84486a41f81c:37329 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-04T10:09:16,586 INFO [84486a41f81c:37329 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-04T10:09:16,586 INFO [84486a41f81c:37329 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-04T10:09:16,586 DEBUG [84486a41f81c:37329 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-04T10:09:16,586 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=84486a41f81c,46309,1733306955769 2024-12-04T10:09:16,588 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 84486a41f81c,46309,1733306955769, state=OPENING 2024-12-04T10:09:16,604 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-04T10:09:16,613 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46309-0x101a11ad9b80001, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T10:09:16,613 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45753-0x101a11ad9b80002, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T10:09:16,613 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37329-0x101a11ad9b80000, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T10:09:16,614 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T10:09:16,614 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T10:09:16,614 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T10:09:16,614 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T10:09:16,614 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=84486a41f81c,46309,1733306955769}] 2024-12-04T10:09:16,766 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T10:09:16,768 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34297, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T10:09:16,770 INFO [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-04T10:09:16,770 INFO [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T10:09:16,771 INFO [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-04T10:09:16,771 INFO [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=84486a41f81c%2C46309%2C1733306955769.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/WALs/84486a41f81c,46309,1733306955769, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/oldWALs, maxLogs=55 2024-12-04T10:09:16,788 INFO [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/WALs/84486a41f81c,46309,1733306955769/84486a41f81c%2C46309%2C1733306955769.meta.1733306956772.meta 2024-12-04T10:09:16,788 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-04T10:09:16,788 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-04T10:09:16,789 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-04T10:09:16,789 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-04T10:09:16,789 INFO [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-04T10:09:16,789 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-04T10:09:16,789 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T10:09:16,789 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-04T10:09:16,789 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-04T10:09:16,790 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-04T10:09:16,791 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T10:09:16,791 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T10:09:16,791 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T10:09:16,791 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-04T10:09:16,791 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T10:09:16,791 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T10:09:16,792 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T10:09:16,792 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-04T10:09:16,792 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T10:09:16,792 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T10:09:16,792 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T10:09:16,792 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-04T10:09:16,793 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T10:09:16,793 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T10:09:16,793 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T10:09:16,793 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T10:09:16,794 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740 2024-12-04T10:09:16,794 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740 2024-12-04T10:09:16,794 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T10:09:16,794 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T10:09:16,794 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T10:09:16,795 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T10:09:16,795 INFO [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67130056, jitterRate=3.1578540802001953E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T10:09:16,796 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-04T10:09:16,796 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733306956789Writing region info on filesystem at 1733306956789Initializing all the Stores at 1733306956790 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733306956790Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733306956790Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733306956790Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733306956790Cleaning up temporary data from old regions at 1733306956794 (+4 ms)Running coprocessor post-open hooks at 1733306956796 (+2 ms)Region opened successfully at 1733306956796 2024-12-04T10:09:16,797 INFO [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733306956765 2024-12-04T10:09:16,799 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-04T10:09:16,799 INFO [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-04T10:09:16,799 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=84486a41f81c,46309,1733306955769 2024-12-04T10:09:16,800 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 84486a41f81c,46309,1733306955769, state=OPEN 2024-12-04T10:09:16,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37329-0x101a11ad9b80000, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T10:09:16,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46309-0x101a11ad9b80001, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T10:09:16,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45753-0x101a11ad9b80002, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T10:09:16,811 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T10:09:16,811 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T10:09:16,811 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T10:09:16,811 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=84486a41f81c,46309,1733306955769 2024-12-04T10:09:16,813 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-04T10:09:16,814 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=84486a41f81c,46309,1733306955769 in 197 msec 2024-12-04T10:09:16,816 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-04T10:09:16,816 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 635 msec 2024-12-04T10:09:16,817 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T10:09:16,817 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-04T10:09:16,821 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T10:09:16,821 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=84486a41f81c,46309,1733306955769, seqNum=-1] 2024-12-04T10:09:16,822 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T10:09:16,824 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35935, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T10:09:16,829 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 695 msec 2024-12-04T10:09:16,829 INFO [master/84486a41f81c:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733306956829, completionTime=-1 2024-12-04T10:09:16,829 INFO [master/84486a41f81c:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-04T10:09:16,829 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-04T10:09:16,831 INFO [master/84486a41f81c:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-04T10:09:16,832 INFO [master/84486a41f81c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733307016832 2024-12-04T10:09:16,832 INFO [master/84486a41f81c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733307076832 2024-12-04T10:09:16,832 INFO [master/84486a41f81c:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-12-04T10:09:16,832 INFO [master/84486a41f81c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=84486a41f81c,37329,1733306955606-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:16,832 INFO [master/84486a41f81c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=84486a41f81c,37329,1733306955606-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:16,832 INFO [master/84486a41f81c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=84486a41f81c,37329,1733306955606-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:16,832 INFO [master/84486a41f81c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-84486a41f81c:37329, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:16,832 INFO [master/84486a41f81c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:16,833 INFO [master/84486a41f81c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:16,835 DEBUG [master/84486a41f81c:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-04T10:09:16,837 INFO [master/84486a41f81c:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.982sec 2024-12-04T10:09:16,838 INFO [master/84486a41f81c:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-04T10:09:16,838 INFO [master/84486a41f81c:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-04T10:09:16,838 INFO [master/84486a41f81c:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-04T10:09:16,838 INFO [master/84486a41f81c:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-04T10:09:16,838 INFO [master/84486a41f81c:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-04T10:09:16,838 INFO [master/84486a41f81c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=84486a41f81c,37329,1733306955606-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T10:09:16,838 INFO [master/84486a41f81c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=84486a41f81c,37329,1733306955606-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-04T10:09:16,839 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-04T10:09:16,839 INFO [master/84486a41f81c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-04T10:09:16,839 INFO [master/84486a41f81c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=84486a41f81c,37329,1733306955606-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:16,923 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d25b74f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T10:09:16,923 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 84486a41f81c,37329,-1 for getting cluster id 2024-12-04T10:09:16,923 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T10:09:16,925 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0117e7a0-3c2e-4ff9-bc50-889e5f40698d' 2024-12-04T10:09:16,925 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T10:09:16,925 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0117e7a0-3c2e-4ff9-bc50-889e5f40698d" 2024-12-04T10:09:16,926 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37db8a65, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T10:09:16,926 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [84486a41f81c,37329,-1] 2024-12-04T10:09:16,926 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T10:09:16,926 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T10:09:16,928 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35644, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T10:09:16,929 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5858d96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T10:09:16,929 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T10:09:16,931 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=84486a41f81c,46309,1733306955769, seqNum=-1] 2024-12-04T10:09:16,931 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T10:09:16,934 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60218, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T10:09:16,936 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=84486a41f81c,37329,1733306955606 2024-12-04T10:09:16,936 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T10:09:16,937 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 84486a41f81c,37329,1733306955606 2024-12-04T10:09:16,937 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@78eb0515 2024-12-04T10:09:16,938 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T10:09:16,940 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35654, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T10:09:16,941 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37329 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1733306956936', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T10:09:16,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37329 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733306956936 2024-12-04T10:09:16,943 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733306956936 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T10:09:16,943 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T10:09:16,943 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37329 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1733306956936" procId is: 4 2024-12-04T10:09:16,944 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733306956936 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T10:09:16,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37329 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T10:09:16,953 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733306956936-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 29794f0ee13f249de2677b8de66b1e0b, NAME => 'testBlockEvictionOnGracefulStop1733306956936,,1733306956940.29794f0ee13f249de2677b8de66b1e0b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1733306956936', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee 2024-12-04T10:09:16,958 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733306956936-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733306956936,,1733306956940.29794f0ee13f249de2677b8de66b1e0b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T10:09:16,959 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733306956936-pool-0 {}] regionserver.HRegion(1722): Closing 29794f0ee13f249de2677b8de66b1e0b, disabling compactions & flushes 2024-12-04T10:09:16,959 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733306956936-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733306956936,,1733306956940.29794f0ee13f249de2677b8de66b1e0b. 2024-12-04T10:09:16,959 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733306956936-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733306956936,,1733306956940.29794f0ee13f249de2677b8de66b1e0b. 2024-12-04T10:09:16,959 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733306956936-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733306956936,,1733306956940.29794f0ee13f249de2677b8de66b1e0b. after waiting 0 ms 2024-12-04T10:09:16,959 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733306956936-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733306956936,,1733306956940.29794f0ee13f249de2677b8de66b1e0b. 2024-12-04T10:09:16,959 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733306956936-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733306956936,,1733306956940.29794f0ee13f249de2677b8de66b1e0b. 2024-12-04T10:09:16,959 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733306956936-pool-0 {}] regionserver.HRegion(1676): Region close journal for 29794f0ee13f249de2677b8de66b1e0b: Waiting for close lock at 1733306956958Disabling compacts and flushes for region at 1733306956958Disabling writes for close at 1733306956959 (+1 ms)Writing region close event to WAL at 1733306956959Closed at 1733306956959 2024-12-04T10:09:16,960 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733306956936 execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T10:09:16,960 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1733306956936,,1733306956940.29794f0ee13f249de2677b8de66b1e0b.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1733306956960"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733306956960"}]},"ts":"1733306956960"} 2024-12-04T10:09:16,962 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T10:09:16,963 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733306956936 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T10:09:16,964 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1733306956936","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733306956963"}]},"ts":"1733306956963"} 2024-12-04T10:09:16,966 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1733306956936, state=ENABLING in hbase:meta 2024-12-04T10:09:16,966 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {84486a41f81c=0} racks are {/default-rack=0} 2024-12-04T10:09:16,966 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-04T10:09:16,966 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-04T10:09:16,966 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-04T10:09:16,967 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-04T10:09:16,967 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-04T10:09:16,967 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-04T10:09:16,967 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-04T10:09:16,967 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733306956936, region=29794f0ee13f249de2677b8de66b1e0b, ASSIGN}] 2024-12-04T10:09:16,968 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733306956936, region=29794f0ee13f249de2677b8de66b1e0b, ASSIGN 2024-12-04T10:09:16,969 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733306956936, region=29794f0ee13f249de2677b8de66b1e0b, ASSIGN; state=OFFLINE, location=84486a41f81c,45753,1733306955802; forceNewPlan=false, retain=false 2024-12-04T10:09:17,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37329 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T10:09:17,119 INFO [84486a41f81c:37329 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-04T10:09:17,120 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=29794f0ee13f249de2677b8de66b1e0b, regionState=OPENING, regionLocation=84486a41f81c,45753,1733306955802 2024-12-04T10:09:17,126 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733306956936, region=29794f0ee13f249de2677b8de66b1e0b, ASSIGN because future has completed 2024-12-04T10:09:17,127 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 29794f0ee13f249de2677b8de66b1e0b, server=84486a41f81c,45753,1733306955802}] 2024-12-04T10:09:17,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37329 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T10:09:17,281 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T10:09:17,284 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55913, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T10:09:17,290 INFO [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1733306956936,,1733306956940.29794f0ee13f249de2677b8de66b1e0b. 2024-12-04T10:09:17,291 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 29794f0ee13f249de2677b8de66b1e0b, NAME => 'testBlockEvictionOnGracefulStop1733306956936,,1733306956940.29794f0ee13f249de2677b8de66b1e0b.', STARTKEY => '', ENDKEY => ''} 2024-12-04T10:09:17,292 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1733306956936 29794f0ee13f249de2677b8de66b1e0b 2024-12-04T10:09:17,293 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733306956936,,1733306956940.29794f0ee13f249de2677b8de66b1e0b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T10:09:17,293 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 29794f0ee13f249de2677b8de66b1e0b 2024-12-04T10:09:17,293 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 29794f0ee13f249de2677b8de66b1e0b 2024-12-04T10:09:17,295 INFO [StoreOpener-29794f0ee13f249de2677b8de66b1e0b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 29794f0ee13f249de2677b8de66b1e0b 2024-12-04T10:09:17,300 INFO [StoreOpener-29794f0ee13f249de2677b8de66b1e0b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 29794f0ee13f249de2677b8de66b1e0b columnFamilyName family 2024-12-04T10:09:17,300 DEBUG [StoreOpener-29794f0ee13f249de2677b8de66b1e0b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T10:09:17,300 INFO [StoreOpener-29794f0ee13f249de2677b8de66b1e0b-1 {}] regionserver.HStore(327): Store=29794f0ee13f249de2677b8de66b1e0b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T10:09:17,300 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 29794f0ee13f249de2677b8de66b1e0b 2024-12-04T10:09:17,301 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/default/testBlockEvictionOnGracefulStop1733306956936/29794f0ee13f249de2677b8de66b1e0b 2024-12-04T10:09:17,301 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/default/testBlockEvictionOnGracefulStop1733306956936/29794f0ee13f249de2677b8de66b1e0b 2024-12-04T10:09:17,301 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 29794f0ee13f249de2677b8de66b1e0b 2024-12-04T10:09:17,301 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 29794f0ee13f249de2677b8de66b1e0b 2024-12-04T10:09:17,302 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 29794f0ee13f249de2677b8de66b1e0b 2024-12-04T10:09:17,307 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/default/testBlockEvictionOnGracefulStop1733306956936/29794f0ee13f249de2677b8de66b1e0b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T10:09:17,308 INFO [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 29794f0ee13f249de2677b8de66b1e0b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73494260, jitterRate=0.09514981508255005}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T10:09:17,308 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 29794f0ee13f249de2677b8de66b1e0b 2024-12-04T10:09:17,309 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 29794f0ee13f249de2677b8de66b1e0b: Running coprocessor pre-open hook at 1733306957293Writing region info on filesystem at 1733306957293Initializing all the Stores at 1733306957294 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733306957294Cleaning up temporary data from old regions at 1733306957301 (+7 ms)Running coprocessor post-open hooks at 1733306957308 (+7 ms)Region opened successfully at 1733306957309 (+1 ms) 2024-12-04T10:09:17,310 INFO [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1733306956936,,1733306956940.29794f0ee13f249de2677b8de66b1e0b., pid=6, masterSystemTime=1733306957280 2024-12-04T10:09:17,313 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1733306956936,,1733306956940.29794f0ee13f249de2677b8de66b1e0b. 2024-12-04T10:09:17,313 INFO [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1733306956936,,1733306956940.29794f0ee13f249de2677b8de66b1e0b. 2024-12-04T10:09:17,313 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=29794f0ee13f249de2677b8de66b1e0b, regionState=OPEN, openSeqNum=2, regionLocation=84486a41f81c,45753,1733306955802 2024-12-04T10:09:17,316 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 29794f0ee13f249de2677b8de66b1e0b, server=84486a41f81c,45753,1733306955802 because future has completed 2024-12-04T10:09:17,320 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-04T10:09:17,320 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 29794f0ee13f249de2677b8de66b1e0b, server=84486a41f81c,45753,1733306955802 in 191 msec 2024-12-04T10:09:17,322 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-04T10:09:17,322 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733306956936, region=29794f0ee13f249de2677b8de66b1e0b, ASSIGN in 353 msec 2024-12-04T10:09:17,323 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733306956936 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T10:09:17,323 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1733306956936","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733306957323"}]},"ts":"1733306957323"} 2024-12-04T10:09:17,326 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1733306956936, state=ENABLED in hbase:meta 2024-12-04T10:09:17,327 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733306956936 execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T10:09:17,328 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733306956936 in 386 msec 2024-12-04T10:09:17,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37329 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T10:09:17,572 INFO [RPCClient-NioEventLoopGroup-5-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1733306956936 completed 2024-12-04T10:09:17,572 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1733306956936 get assigned. Timeout = 60000ms 2024-12-04T10:09:17,573 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T10:09:17,580 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1733306956936 assigned to meta. Checking AM states. 2024-12-04T10:09:17,580 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T10:09:17,581 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1733306956936 assigned. 2024-12-04T10:09:17,586 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1733306956936', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1733306956936,,1733306956940.29794f0ee13f249de2677b8de66b1e0b., hostname=84486a41f81c,45753,1733306955802, seqNum=2] 2024-12-04T10:09:17,587 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T10:09:17,589 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37222, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T10:09:17,593 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 29794f0ee13f249de2677b8de66b1e0b 1/1 column families, dataSize=78 B heapSize=480 B 2024-12-04T10:09:17,600 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 945f1620af7d490989a9cdf442b53ea5 2024-12-04T10:09:17,612 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/default/testBlockEvictionOnGracefulStop1733306956936/29794f0ee13f249de2677b8de66b1e0b/.tmp/family/945f1620af7d490989a9cdf442b53ea5 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-12-04T10:09:17,613 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/default/testBlockEvictionOnGracefulStop1733306956936/29794f0ee13f249de2677b8de66b1e0b/.tmp/family/945f1620af7d490989a9cdf442b53ea5 2024-12-04T10:09:17,616 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/default/testBlockEvictionOnGracefulStop1733306956936/29794f0ee13f249de2677b8de66b1e0b/.tmp/family/945f1620af7d490989a9cdf442b53ea5 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/default/testBlockEvictionOnGracefulStop1733306956936/29794f0ee13f249de2677b8de66b1e0b/family/945f1620af7d490989a9cdf442b53ea5 2024-12-04T10:09:17,618 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/default/testBlockEvictionOnGracefulStop1733306956936/29794f0ee13f249de2677b8de66b1e0b/family/945f1620af7d490989a9cdf442b53ea5, delay=970 ms 2024-12-04T10:09:17,619 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/default/testBlockEvictionOnGracefulStop1733306956936/29794f0ee13f249de2677b8de66b1e0b/family/945f1620af7d490989a9cdf442b53ea5, entries=2, sequenceid=6, filesize=4.9 K 2024-12-04T10:09:17,620 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 29794f0ee13f249de2677b8de66b1e0b in 27ms, sequenceid=6, compaction requested=false 2024-12-04T10:09:17,620 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1733306956936' 2024-12-04T10:09:17,621 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 29794f0ee13f249de2677b8de66b1e0b: 2024-12-04T10:09:17,893 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T10:09:18,589 DEBUG [hfile-prefetch-1733306954984 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 945f1620af7d490989a9cdf442b53ea5_0. 2024-12-04T10:09:18,589 DEBUG [hfile-prefetch-1733306954984 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 945f1620af7d490989a9cdf442b53ea5_117. 2024-12-04T10:09:18,590 DEBUG [hfile-prefetch-1733306954984 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/default/testBlockEvictionOnGracefulStop1733306956936/29794f0ee13f249de2677b8de66b1e0b/family/945f1620af7d490989a9cdf442b53ea5, with total blocks 2, and data blocks 1 2024-12-04T10:09:18,590 DEBUG [hfile-prefetch-1733306954984 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-04T10:09:18,590 DEBUG [hfile-prefetch-1733306954984 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/default/testBlockEvictionOnGracefulStop1733306956936/29794f0ee13f249de2677b8de66b1e0b/family/945f1620af7d490989a9cdf442b53ea5, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-12-04T10:09:18,590 DEBUG [hfile-prefetch-1733306954984 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/default/testBlockEvictionOnGracefulStop1733306956936/29794f0ee13f249de2677b8de66b1e0b/family/945f1620af7d490989a9cdf442b53ea5. Will sleep 100ms and try the verification again. 2024-12-04T10:09:18,622 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;84486a41f81c:45753,5,FailOnTimeoutGroup] 2024-12-04T10:09:18,622 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '84486a41f81c,45753,1733306955802' ***** 2024-12-04T10:09:18,622 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-12-04T10:09:18,622 INFO [RS:1;84486a41f81c:45753 {}] bucket.BucketCache(1738): Disabling cache 2024-12-04T10:09:18,631 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-04T10:09:18,631 INFO [regionserver/84486a41f81c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T10:09:18,647 INFO [RS:1;84486a41f81c:45753 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d829fef9-ca4a-35f8-a2bc-ee23e1db7a16/bucket.persistence 2024-12-04T10:09:18,647 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-04T10:09:18,647 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-04T10:09:18,647 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-04T10:09:18,647 INFO [RS:1;84486a41f81c:45753 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-04T10:09:18,648 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-04T10:09:18,658 DEBUG [RS:1;84486a41f81c:45753 {}] bucket.BucketCache(1487): Thread RS:1;84486a41f81c:45753 started persisting bucket cache to file 2024-12-04T10:09:18,658 DEBUG [RS:1;84486a41f81c:45753 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-04T10:09:18,658 DEBUG [RS:1;84486a41f81c:45753 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-12-04T10:09:18,665 DEBUG [RS:1;84486a41f81c:45753 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-12-04T10:09:18,665 DEBUG [RS:1;84486a41f81c:45753 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1733306958658 2024-12-04T10:09:18,665 DEBUG [RS:1;84486a41f81c:45753 {}] bucket.BucketCache(1509): Thread RS:1;84486a41f81c:45753 finished persisting bucket cache to file, renaming 2024-12-04T10:09:18,666 INFO [RS:1;84486a41f81c:45753 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T10:09:18,666 INFO [RS:1;84486a41f81c:45753 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-04T10:09:18,666 INFO [RS:1;84486a41f81c:45753 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-04T10:09:18,666 INFO [RS:1;84486a41f81c:45753 {}] regionserver.HRegionServer(3091): Received CLOSE for 29794f0ee13f249de2677b8de66b1e0b 2024-12-04T10:09:18,666 INFO [RS:1;84486a41f81c:45753 {}] regionserver.HRegionServer(959): stopping server 84486a41f81c,45753,1733306955802 2024-12-04T10:09:18,666 INFO [RS:1;84486a41f81c:45753 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T10:09:18,666 INFO [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1733306956936,,1733306956940.29794f0ee13f249de2677b8de66b1e0b. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-04T10:09:18,667 INFO [RS:1;84486a41f81c:45753 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;84486a41f81c:45753. 2024-12-04T10:09:18,667 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 29794f0ee13f249de2677b8de66b1e0b, disabling compactions & flushes 2024-12-04T10:09:18,667 INFO [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733306956936,,1733306956940.29794f0ee13f249de2677b8de66b1e0b. 2024-12-04T10:09:18,667 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733306956936,,1733306956940.29794f0ee13f249de2677b8de66b1e0b. 2024-12-04T10:09:18,667 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733306956936,,1733306956940.29794f0ee13f249de2677b8de66b1e0b. after waiting 0 ms 2024-12-04T10:09:18,667 DEBUG [RS:1;84486a41f81c:45753 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T10:09:18,667 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733306956936,,1733306956940.29794f0ee13f249de2677b8de66b1e0b. 2024-12-04T10:09:18,667 DEBUG [RS:1;84486a41f81c:45753 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T10:09:18,667 INFO [RS:1;84486a41f81c:45753 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-04T10:09:18,667 DEBUG [RS:1;84486a41f81c:45753 {}] regionserver.HRegionServer(1325): Online Regions={29794f0ee13f249de2677b8de66b1e0b=testBlockEvictionOnGracefulStop1733306956936,,1733306956940.29794f0ee13f249de2677b8de66b1e0b.} 2024-12-04T10:09:18,667 DEBUG [RS:1;84486a41f81c:45753 {}] regionserver.HRegionServer(1351): Waiting on 29794f0ee13f249de2677b8de66b1e0b 2024-12-04T10:09:18,668 DEBUG [StoreFileCloser-29794f0ee13f249de2677b8de66b1e0b-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/default/testBlockEvictionOnGracefulStop1733306956936/29794f0ee13f249de2677b8de66b1e0b/family/945f1620af7d490989a9cdf442b53ea5 2024-12-04T10:09:18,668 WARN [hfile-prefetch-1733306954984 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/default/testBlockEvictionOnGracefulStop1733306956936/29794f0ee13f249de2677b8de66b1e0b/family/945f1620af7d490989a9cdf442b53ea5, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-12-04T10:09:18,668 DEBUG [hfile-prefetch-1733306954984 {}] hfile.PrefetchExecutor(121): Prefetch completed for 945f1620af7d490989a9cdf442b53ea5 2024-12-04T10:09:18,672 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/default/testBlockEvictionOnGracefulStop1733306956936/29794f0ee13f249de2677b8de66b1e0b/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-04T10:09:18,673 INFO [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733306956936,,1733306956940.29794f0ee13f249de2677b8de66b1e0b. 2024-12-04T10:09:18,673 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 29794f0ee13f249de2677b8de66b1e0b: Waiting for close lock at 1733306958666Running coprocessor pre-close hooks at 1733306958667 (+1 ms)Disabling compacts and flushes for region at 1733306958667Disabling writes for close at 1733306958667Writing region close event to WAL at 1733306958668 (+1 ms)Running coprocessor post-close hooks at 1733306958673 (+5 ms)Closed at 1733306958673 2024-12-04T10:09:18,673 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1733306956936,,1733306956940.29794f0ee13f249de2677b8de66b1e0b. 2024-12-04T10:09:18,684 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-04T10:09:18,684 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-04T10:09:18,685 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-04T10:09:18,685 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-04T10:09:18,687 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-04T10:09:18,688 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733306958684 2024-12-04T10:09:18,688 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-04T10:09:18,868 INFO [RS:1;84486a41f81c:45753 {}] regionserver.HRegionServer(976): stopping server 84486a41f81c,45753,1733306955802; all regions closed. 2024-12-04T10:09:18,874 DEBUG [RS:1;84486a41f81c:45753 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/oldWALs 2024-12-04T10:09:18,874 INFO [RS:1;84486a41f81c:45753 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 84486a41f81c%2C45753%2C1733306955802:(num 1733306956460) 2024-12-04T10:09:18,874 DEBUG [RS:1;84486a41f81c:45753 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T10:09:18,875 INFO [RS:1;84486a41f81c:45753 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T10:09:18,875 INFO [RS:1;84486a41f81c:45753 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T10:09:18,875 INFO [RS:1;84486a41f81c:45753 {}] hbase.ChoreService(370): Chore service for: regionserver/84486a41f81c:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-04T10:09:18,876 INFO [RS:1;84486a41f81c:45753 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T10:09:18,876 INFO [regionserver/84486a41f81c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T10:09:18,876 INFO [RS:1;84486a41f81c:45753 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T10:09:18,876 INFO [RS:1;84486a41f81c:45753 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T10:09:18,876 INFO [RS:1;84486a41f81c:45753 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T10:09:18,876 INFO [RS:1;84486a41f81c:45753 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45753 2024-12-04T10:09:18,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45753-0x101a11ad9b80002, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/84486a41f81c,45753,1733306955802 2024-12-04T10:09:18,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37329-0x101a11ad9b80000, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T10:09:18,896 INFO [RS:1;84486a41f81c:45753 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T10:09:18,905 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [84486a41f81c,45753,1733306955802] 2024-12-04T10:09:18,913 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/84486a41f81c,45753,1733306955802 already deleted, retry=false 2024-12-04T10:09:18,913 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 84486a41f81c,45753,1733306955802 on 84486a41f81c,37329,1733306955606 2024-12-04T10:09:18,919 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 84486a41f81c,45753,1733306955802, splitWal=true, meta=false 2024-12-04T10:09:18,922 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 84486a41f81c,45753,1733306955802, splitWal=true, meta=false 2024-12-04T10:09:18,922 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 84486a41f81c,45753,1733306955802 (carryingMeta=false) 84486a41f81c,45753,1733306955802/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@13ccf89[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-04T10:09:18,924 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 84486a41f81c,45753,1733306955802 had 1 regions 2024-12-04T10:09:18,925 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 84486a41f81c,45753,1733306955802, splitWal=true, meta=false, isMeta: false 2024-12-04T10:09:18,926 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/WALs/84486a41f81c,45753,1733306955802-splitting 2024-12-04T10:09:18,926 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/WALs/84486a41f81c,45753,1733306955802-splitting dir is empty, no logs to split. 2024-12-04T10:09:18,926 INFO [PEWorker-1 {}] master.SplitWALManager(105): 84486a41f81c,45753,1733306955802 WAL count=0, meta=false 2024-12-04T10:09:18,928 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/WALs/84486a41f81c,45753,1733306955802-splitting dir is empty, no logs to split. 2024-12-04T10:09:18,928 INFO [PEWorker-1 {}] master.SplitWALManager(105): 84486a41f81c,45753,1733306955802 WAL count=0, meta=false 2024-12-04T10:09:18,928 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 84486a41f81c,45753,1733306955802 WAL splitting is done? wals=0, meta=false 2024-12-04T10:09:18,928 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 84486a41f81c,45753,1733306955802 failed, ignore...File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/WALs/84486a41f81c,45753,1733306955802-splitting does not exist 2024-12-04T10:09:18,929 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733306956936, region=29794f0ee13f249de2677b8de66b1e0b, ASSIGN}] 2024-12-04T10:09:18,930 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733306956936, region=29794f0ee13f249de2677b8de66b1e0b, ASSIGN 2024-12-04T10:09:18,931 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733306956936, region=29794f0ee13f249de2677b8de66b1e0b, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-04T10:09:19,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45753-0x101a11ad9b80002, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T10:09:19,005 INFO [RS:1;84486a41f81c:45753 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T10:09:19,005 INFO [RS:1;84486a41f81c:45753 {}] regionserver.HRegionServer(1031): Exiting; stopping=84486a41f81c,45753,1733306955802; zookeeper connection closed. 2024-12-04T10:09:19,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45753-0x101a11ad9b80002, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T10:09:19,005 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@675f7311 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@675f7311 2024-12-04T10:09:19,082 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=29794f0ee13f249de2677b8de66b1e0b, regionState=OPENING, regionLocation=84486a41f81c,46309,1733306955769 2024-12-04T10:09:19,089 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733306956936, region=29794f0ee13f249de2677b8de66b1e0b, ASSIGN because future has completed 2024-12-04T10:09:19,090 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 29794f0ee13f249de2677b8de66b1e0b, server=84486a41f81c,46309,1733306955769}] 2024-12-04T10:09:19,139 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/84486a41f81c:0 server-side Connection retries=45 2024-12-04T10:09:19,139 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T10:09:19,139 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T10:09:19,139 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T10:09:19,139 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T10:09:19,139 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T10:09:19,139 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T10:09:19,139 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T10:09:19,140 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44233 2024-12-04T10:09:19,141 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44233 connecting to ZooKeeper ensemble=127.0.0.1:57330 2024-12-04T10:09:19,141 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T10:09:19,141 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T10:09:19,152 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:442330x0, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T10:09:19,153 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44233-0x101a11ad9b80003 connected 2024-12-04T10:09:19,153 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:44233-0x101a11ad9b80003, quorum=127.0.0.1:57330, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-12-04T10:09:19,153 DEBUG [pool-297-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-12-04T10:09:19,154 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T10:09:19,154 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d829fef9-ca4a-35f8-a2bc-ee23e1db7a16/bucket.cache 2024-12-04T10:09:19,154 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-04T10:09:19,154 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-04T10:09:19,154 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-04T10:09:19,155 INFO [Thread-199 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-04T10:09:19,155 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d829fef9-ca4a-35f8-a2bc-ee23e1db7a16/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d829fef9-ca4a-35f8-a2bc-ee23e1db7a16/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-04T10:09:19,155 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T10:09:19,156 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:44233-0x101a11ad9b80003, quorum=127.0.0.1:57330, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-04T10:09:19,156 INFO [Thread-199 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-12-04T10:09:19,156 INFO [Thread-199 {}] bucket.BucketCache(1705): Retrieved 0 of chunks with blockCount = 0. 2024-12-04T10:09:19,158 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44233-0x101a11ad9b80003, quorum=127.0.0.1:57330, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T10:09:19,162 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44233 2024-12-04T10:09:19,163 DEBUG [Thread-199 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-12-04T10:09:19,163 INFO [Thread-199 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-04T10:09:19,163 INFO [Thread-199 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 0 2024-12-04T10:09:19,163 INFO [Thread-199 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d829fef9-ca4a-35f8-a2bc-ee23e1db7a16/bucket.persistence is complete. 2024-12-04T10:09:19,163 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44233 2024-12-04T10:09:19,167 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44233 2024-12-04T10:09:19,167 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44233 2024-12-04T10:09:19,170 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44233 2024-12-04T10:09:19,172 INFO [RS:2;84486a41f81c:44233 {}] regionserver.HRegionServer(746): ClusterId : 0117e7a0-3c2e-4ff9-bc50-889e5f40698d 2024-12-04T10:09:19,172 DEBUG [RS:2;84486a41f81c:44233 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T10:09:19,245 DEBUG [RS:2;84486a41f81c:44233 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T10:09:19,245 INFO [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1733306956936,,1733306956940.29794f0ee13f249de2677b8de66b1e0b. 2024-12-04T10:09:19,245 DEBUG [RS:2;84486a41f81c:44233 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T10:09:19,245 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 29794f0ee13f249de2677b8de66b1e0b, NAME => 'testBlockEvictionOnGracefulStop1733306956936,,1733306956940.29794f0ee13f249de2677b8de66b1e0b.', STARTKEY => '', ENDKEY => ''} 2024-12-04T10:09:19,246 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1733306956936 29794f0ee13f249de2677b8de66b1e0b 2024-12-04T10:09:19,246 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733306956936,,1733306956940.29794f0ee13f249de2677b8de66b1e0b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T10:09:19,246 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 29794f0ee13f249de2677b8de66b1e0b 2024-12-04T10:09:19,246 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 29794f0ee13f249de2677b8de66b1e0b 2024-12-04T10:09:19,247 INFO [StoreOpener-29794f0ee13f249de2677b8de66b1e0b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 29794f0ee13f249de2677b8de66b1e0b 2024-12-04T10:09:19,247 INFO [StoreOpener-29794f0ee13f249de2677b8de66b1e0b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 29794f0ee13f249de2677b8de66b1e0b columnFamilyName family 2024-12-04T10:09:19,247 DEBUG [StoreOpener-29794f0ee13f249de2677b8de66b1e0b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T10:09:19,249 DEBUG [StoreFileOpener-29794f0ee13f249de2677b8de66b1e0b-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/default/testBlockEvictionOnGracefulStop1733306956936/29794f0ee13f249de2677b8de66b1e0b/family/945f1620af7d490989a9cdf442b53ea5, delay=964 ms 2024-12-04T10:09:19,250 DEBUG [StoreOpener-29794f0ee13f249de2677b8de66b1e0b-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/default/testBlockEvictionOnGracefulStop1733306956936/29794f0ee13f249de2677b8de66b1e0b/family/945f1620af7d490989a9cdf442b53ea5 2024-12-04T10:09:19,250 INFO [StoreOpener-29794f0ee13f249de2677b8de66b1e0b-1 {}] regionserver.HStore(327): Store=29794f0ee13f249de2677b8de66b1e0b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T10:09:19,250 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 29794f0ee13f249de2677b8de66b1e0b 2024-12-04T10:09:19,251 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/default/testBlockEvictionOnGracefulStop1733306956936/29794f0ee13f249de2677b8de66b1e0b 2024-12-04T10:09:19,251 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/default/testBlockEvictionOnGracefulStop1733306956936/29794f0ee13f249de2677b8de66b1e0b 2024-12-04T10:09:19,251 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 29794f0ee13f249de2677b8de66b1e0b 2024-12-04T10:09:19,251 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 29794f0ee13f249de2677b8de66b1e0b 2024-12-04T10:09:19,252 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 29794f0ee13f249de2677b8de66b1e0b 2024-12-04T10:09:19,253 INFO [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 29794f0ee13f249de2677b8de66b1e0b; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74137476, jitterRate=0.10473448038101196}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T10:09:19,253 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 29794f0ee13f249de2677b8de66b1e0b 2024-12-04T10:09:19,254 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 29794f0ee13f249de2677b8de66b1e0b: Running coprocessor pre-open hook at 1733306959246Writing region info on filesystem at 1733306959246Initializing all the Stores at 1733306959246Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733306959246Cleaning up temporary data from old regions at 1733306959251 (+5 ms)Running coprocessor post-open hooks at 1733306959253 (+2 ms)Region opened successfully at 1733306959254 (+1 ms) 2024-12-04T10:09:19,255 INFO [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1733306956936,,1733306956940.29794f0ee13f249de2677b8de66b1e0b., pid=9, masterSystemTime=1733306959242 2024-12-04T10:09:19,255 DEBUG [RS:2;84486a41f81c:44233 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T10:09:19,256 DEBUG [RS:2;84486a41f81c:44233 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dd5a6cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=84486a41f81c/172.17.0.2:0 2024-12-04T10:09:19,258 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1733306956936,,1733306956940.29794f0ee13f249de2677b8de66b1e0b. 2024-12-04T10:09:19,258 INFO [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1733306956936,,1733306956940.29794f0ee13f249de2677b8de66b1e0b. 2024-12-04T10:09:19,259 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=29794f0ee13f249de2677b8de66b1e0b, regionState=OPEN, openSeqNum=10, regionLocation=84486a41f81c,46309,1733306955769 2024-12-04T10:09:19,261 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 29794f0ee13f249de2677b8de66b1e0b, server=84486a41f81c,46309,1733306955769 because future has completed 2024-12-04T10:09:19,264 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-12-04T10:09:19,264 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 29794f0ee13f249de2677b8de66b1e0b, server=84486a41f81c,46309,1733306955769 in 173 msec 2024-12-04T10:09:19,266 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-04T10:09:19,266 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(291): removed crashed server 84486a41f81c,45753,1733306955802 after splitting done 2024-12-04T10:09:19,266 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733306956936, region=29794f0ee13f249de2677b8de66b1e0b, ASSIGN in 335 msec 2024-12-04T10:09:19,267 DEBUG [RS:2;84486a41f81c:44233 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;84486a41f81c:44233 2024-12-04T10:09:19,268 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 84486a41f81c,45753,1733306955802, splitWal=true, meta=false in 352 msec 2024-12-04T10:09:19,268 INFO [RS:2;84486a41f81c:44233 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-04T10:09:19,268 INFO [RS:2;84486a41f81c:44233 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-04T10:09:19,268 DEBUG [RS:2;84486a41f81c:44233 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-04T10:09:19,268 INFO [RS:2;84486a41f81c:44233 {}] regionserver.HRegionServer(2659): reportForDuty to master=84486a41f81c,37329,1733306955606 with port=44233, startcode=1733306959138 2024-12-04T10:09:19,268 DEBUG [RS:2;84486a41f81c:44233 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T10:09:19,270 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51611, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T10:09:19,271 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37329 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 84486a41f81c,44233,1733306959138 2024-12-04T10:09:19,271 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37329 {}] master.ServerManager(517): Registering regionserver=84486a41f81c,44233,1733306959138 2024-12-04T10:09:19,272 DEBUG [RS:2;84486a41f81c:44233 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee 2024-12-04T10:09:19,272 DEBUG [RS:2;84486a41f81c:44233 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-04T10:09:19,272 DEBUG [RS:2;84486a41f81c:44233 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-04T10:09:19,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37329-0x101a11ad9b80000, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T10:09:19,280 DEBUG [RS:2;84486a41f81c:44233 {}] zookeeper.ZKUtil(111): regionserver:44233-0x101a11ad9b80003, quorum=127.0.0.1:57330, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/84486a41f81c,44233,1733306959138 2024-12-04T10:09:19,280 WARN [RS:2;84486a41f81c:44233 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T10:09:19,280 INFO [RS:2;84486a41f81c:44233 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T10:09:19,280 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [84486a41f81c,44233,1733306959138] 2024-12-04T10:09:19,280 DEBUG [RS:2;84486a41f81c:44233 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/WALs/84486a41f81c,44233,1733306959138 2024-12-04T10:09:19,285 INFO [RS:2;84486a41f81c:44233 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T10:09:19,287 INFO [RS:2;84486a41f81c:44233 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T10:09:19,288 INFO [RS:2;84486a41f81c:44233 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T10:09:19,288 INFO [RS:2;84486a41f81c:44233 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:19,288 INFO [RS:2;84486a41f81c:44233 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-04T10:09:19,288 INFO [RS:2;84486a41f81c:44233 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-04T10:09:19,288 INFO [RS:2;84486a41f81c:44233 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:19,288 DEBUG [RS:2;84486a41f81c:44233 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:19,288 DEBUG [RS:2;84486a41f81c:44233 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:19,289 DEBUG [RS:2;84486a41f81c:44233 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:19,289 DEBUG [RS:2;84486a41f81c:44233 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:19,289 DEBUG [RS:2;84486a41f81c:44233 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:19,289 DEBUG [RS:2;84486a41f81c:44233 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/84486a41f81c:0, corePoolSize=2, maxPoolSize=2 2024-12-04T10:09:19,289 DEBUG [RS:2;84486a41f81c:44233 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:19,289 DEBUG [RS:2;84486a41f81c:44233 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:19,289 DEBUG [RS:2;84486a41f81c:44233 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:19,289 DEBUG [RS:2;84486a41f81c:44233 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:19,289 DEBUG [RS:2;84486a41f81c:44233 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:19,289 DEBUG [RS:2;84486a41f81c:44233 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T10:09:19,289 DEBUG [RS:2;84486a41f81c:44233 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/84486a41f81c:0, corePoolSize=3, maxPoolSize=3 2024-12-04T10:09:19,289 DEBUG [RS:2;84486a41f81c:44233 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/84486a41f81c:0, corePoolSize=3, maxPoolSize=3 2024-12-04T10:09:19,290 INFO [RS:2;84486a41f81c:44233 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:19,290 INFO [RS:2;84486a41f81c:44233 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:19,290 INFO [RS:2;84486a41f81c:44233 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:19,290 INFO [RS:2;84486a41f81c:44233 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:19,290 INFO [RS:2;84486a41f81c:44233 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:19,290 INFO [RS:2;84486a41f81c:44233 {}] hbase.ChoreService(168): Chore ScheduledChore name=84486a41f81c,44233,1733306959138-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T10:09:19,304 INFO [RS:2;84486a41f81c:44233 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T10:09:19,304 INFO [RS:2;84486a41f81c:44233 {}] hbase.ChoreService(168): Chore ScheduledChore name=84486a41f81c,44233,1733306959138-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:19,304 INFO [RS:2;84486a41f81c:44233 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:19,304 INFO [RS:2;84486a41f81c:44233 {}] regionserver.Replication(171): 84486a41f81c,44233,1733306959138 started 2024-12-04T10:09:19,317 INFO [RS:2;84486a41f81c:44233 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T10:09:19,317 INFO [RS:2;84486a41f81c:44233 {}] regionserver.HRegionServer(1482): Serving as 84486a41f81c,44233,1733306959138, RpcServer on 84486a41f81c/172.17.0.2:44233, sessionid=0x101a11ad9b80003 2024-12-04T10:09:19,318 DEBUG [RS:2;84486a41f81c:44233 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T10:09:19,318 DEBUG [RS:2;84486a41f81c:44233 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 84486a41f81c,44233,1733306959138 2024-12-04T10:09:19,318 DEBUG [RS:2;84486a41f81c:44233 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '84486a41f81c,44233,1733306959138' 2024-12-04T10:09:19,318 DEBUG [RS:2;84486a41f81c:44233 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T10:09:19,318 DEBUG [RS:2;84486a41f81c:44233 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T10:09:19,319 DEBUG [RS:2;84486a41f81c:44233 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T10:09:19,319 DEBUG [RS:2;84486a41f81c:44233 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T10:09:19,319 DEBUG [RS:2;84486a41f81c:44233 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 84486a41f81c,44233,1733306959138 2024-12-04T10:09:19,319 DEBUG [RS:2;84486a41f81c:44233 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '84486a41f81c,44233,1733306959138' 2024-12-04T10:09:19,319 DEBUG [RS:2;84486a41f81c:44233 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T10:09:19,319 DEBUG [RS:2;84486a41f81c:44233 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T10:09:19,320 DEBUG [RS:2;84486a41f81c:44233 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T10:09:19,320 INFO [RS:2;84486a41f81c:44233 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T10:09:19,320 INFO [RS:2;84486a41f81c:44233 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T10:09:19,421 INFO [RS:2;84486a41f81c:44233 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T10:09:19,421 INFO [RS:2;84486a41f81c:44233 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=84486a41f81c%2C44233%2C1733306959138, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/WALs/84486a41f81c,44233,1733306959138, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/oldWALs, maxLogs=55 2024-12-04T10:09:19,437 INFO [RS:2;84486a41f81c:44233 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/WALs/84486a41f81c,44233,1733306959138/84486a41f81c%2C44233%2C1733306959138.1733306959422 2024-12-04T10:09:19,437 DEBUG [RS:2;84486a41f81c:44233 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-04T10:09:19,818 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-04T10:09:19,818 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T10:09:19,818 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T10:09:19,818 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T10:09:19,818 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T10:09:19,818 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T10:09:19,818 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-04T10:09:19,819 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1432894708, stopped=false 2024-12-04T10:09:19,819 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=84486a41f81c,37329,1733306955606 2024-12-04T10:09:19,861 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46309-0x101a11ad9b80001, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T10:09:19,861 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44233-0x101a11ad9b80003, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T10:09:19,861 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37329-0x101a11ad9b80000, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T10:09:19,861 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46309-0x101a11ad9b80001, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T10:09:19,861 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T10:09:19,861 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44233-0x101a11ad9b80003, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T10:09:19,861 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37329-0x101a11ad9b80000, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T10:09:19,862 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T10:09:19,862 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T10:09:19,862 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T10:09:19,863 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37329-0x101a11ad9b80000, quorum=127.0.0.1:57330, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T10:09:19,863 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '84486a41f81c,46309,1733306955769' ***** 2024-12-04T10:09:19,863 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44233-0x101a11ad9b80003, quorum=127.0.0.1:57330, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T10:09:19,863 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-04T10:09:19,863 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46309-0x101a11ad9b80001, quorum=127.0.0.1:57330, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T10:09:19,863 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '84486a41f81c,44233,1733306959138' ***** 2024-12-04T10:09:19,863 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-04T10:09:19,864 INFO [RS:2;84486a41f81c:44233 {}] bucket.BucketCache(1738): Disabling cache 2024-12-04T10:09:19,864 INFO [RS:0;84486a41f81c:46309 {}] bucket.BucketCache(1738): Disabling cache 2024-12-04T10:09:19,865 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-04T10:09:19,865 INFO [RS:0;84486a41f81c:46309 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d829fef9-ca4a-35f8-a2bc-ee23e1db7a16/bucket.persistence 2024-12-04T10:09:19,865 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-04T10:09:19,865 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-04T10:09:19,865 INFO [RS:0;84486a41f81c:46309 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-04T10:09:19,865 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-04T10:09:19,865 INFO [RS:2;84486a41f81c:44233 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d829fef9-ca4a-35f8-a2bc-ee23e1db7a16/bucket.persistence 2024-12-04T10:09:19,865 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-04T10:09:19,865 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-04T10:09:19,866 INFO [RS:2;84486a41f81c:44233 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-04T10:09:19,866 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-04T10:09:19,866 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-04T10:09:19,866 DEBUG [RS:2;84486a41f81c:44233 {}] bucket.BucketCache(1487): Thread RS:2;84486a41f81c:44233 started persisting bucket cache to file 2024-12-04T10:09:19,867 DEBUG [RS:2;84486a41f81c:44233 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-04T10:09:19,867 DEBUG [RS:2;84486a41f81c:44233 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-04T10:09:19,871 DEBUG [RS:2;84486a41f81c:44233 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-04T10:09:19,871 DEBUG [RS:2;84486a41f81c:44233 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733306959866 2024-12-04T10:09:19,871 DEBUG [RS:2;84486a41f81c:44233 {}] bucket.BucketCache(1509): Thread RS:2;84486a41f81c:44233 finished persisting bucket cache to file, renaming 2024-12-04T10:09:19,871 INFO [RS:2;84486a41f81c:44233 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T10:09:19,871 INFO [RS:2;84486a41f81c:44233 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-04T10:09:19,871 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-04T10:09:19,871 INFO [RS:2;84486a41f81c:44233 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-04T10:09:19,871 INFO [RS:2;84486a41f81c:44233 {}] regionserver.HRegionServer(959): stopping server 84486a41f81c,44233,1733306959138 2024-12-04T10:09:19,871 INFO [RS:2;84486a41f81c:44233 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T10:09:19,871 INFO [RS:2;84486a41f81c:44233 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;84486a41f81c:44233. 2024-12-04T10:09:19,872 DEBUG [RS:2;84486a41f81c:44233 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T10:09:19,872 DEBUG [RS:2;84486a41f81c:44233 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T10:09:19,872 INFO [RS:2;84486a41f81c:44233 {}] regionserver.HRegionServer(976): stopping server 84486a41f81c,44233,1733306959138; all regions closed. 2024-12-04T10:09:19,874 DEBUG [RS:2;84486a41f81c:44233 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/oldWALs 2024-12-04T10:09:19,874 INFO [RS:2;84486a41f81c:44233 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 84486a41f81c%2C44233%2C1733306959138:(num 1733306959422) 2024-12-04T10:09:19,874 DEBUG [RS:2;84486a41f81c:44233 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T10:09:19,874 INFO [RS:2;84486a41f81c:44233 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T10:09:19,874 INFO [RS:2;84486a41f81c:44233 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T10:09:19,874 INFO [RS:2;84486a41f81c:44233 {}] hbase.ChoreService(370): Chore service for: regionserver/84486a41f81c:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-04T10:09:19,875 INFO [RS:2;84486a41f81c:44233 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T10:09:19,875 INFO [regionserver/84486a41f81c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T10:09:19,875 INFO [RS:2;84486a41f81c:44233 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T10:09:19,875 INFO [RS:2;84486a41f81c:44233 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T10:09:19,875 INFO [RS:2;84486a41f81c:44233 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T10:09:19,875 INFO [RS:2;84486a41f81c:44233 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44233 2024-12-04T10:09:19,875 DEBUG [RS:0;84486a41f81c:46309 {}] bucket.BucketCache(1487): Thread RS:0;84486a41f81c:46309 started persisting bucket cache to file 2024-12-04T10:09:19,876 DEBUG [RS:0;84486a41f81c:46309 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-04T10:09:19,876 DEBUG [RS:0;84486a41f81c:46309 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-04T10:09:19,878 DEBUG [RS:0;84486a41f81c:46309 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-04T10:09:19,878 DEBUG [RS:0;84486a41f81c:46309 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733306959876 2024-12-04T10:09:19,878 DEBUG [RS:0;84486a41f81c:46309 {}] bucket.BucketCache(1509): Thread RS:0;84486a41f81c:46309 finished persisting bucket cache to file, renaming 2024-12-04T10:09:19,878 INFO [RS:0;84486a41f81c:46309 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T10:09:19,879 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-04T10:09:19,879 INFO [RS:0;84486a41f81c:46309 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-04T10:09:19,879 INFO [RS:0;84486a41f81c:46309 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-04T10:09:19,879 INFO [RS:0;84486a41f81c:46309 {}] regionserver.HRegionServer(3091): Received CLOSE for 29794f0ee13f249de2677b8de66b1e0b 2024-12-04T10:09:19,879 INFO [RS:0;84486a41f81c:46309 {}] regionserver.HRegionServer(959): stopping server 84486a41f81c,46309,1733306955769 2024-12-04T10:09:19,879 INFO [RS:0;84486a41f81c:46309 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T10:09:19,879 INFO [RS:0;84486a41f81c:46309 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;84486a41f81c:46309. 2024-12-04T10:09:19,879 DEBUG [RS:0;84486a41f81c:46309 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T10:09:19,879 DEBUG [RS:0;84486a41f81c:46309 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T10:09:19,879 INFO [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1733306956936,,1733306956940.29794f0ee13f249de2677b8de66b1e0b. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-04T10:09:19,879 INFO [RS:0;84486a41f81c:46309 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T10:09:19,879 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 29794f0ee13f249de2677b8de66b1e0b, disabling compactions & flushes 2024-12-04T10:09:19,879 INFO [RS:0;84486a41f81c:46309 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T10:09:19,879 INFO [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733306956936,,1733306956940.29794f0ee13f249de2677b8de66b1e0b. 2024-12-04T10:09:19,879 INFO [RS:0;84486a41f81c:46309 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T10:09:19,879 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733306956936,,1733306956940.29794f0ee13f249de2677b8de66b1e0b. 2024-12-04T10:09:19,879 INFO [RS:0;84486a41f81c:46309 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-04T10:09:19,879 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733306956936,,1733306956940.29794f0ee13f249de2677b8de66b1e0b. after waiting 0 ms 2024-12-04T10:09:19,879 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733306956936,,1733306956940.29794f0ee13f249de2677b8de66b1e0b. 2024-12-04T10:09:19,880 INFO [RS:0;84486a41f81c:46309 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-04T10:09:19,880 DEBUG [RS:0;84486a41f81c:46309 {}] regionserver.HRegionServer(1325): Online Regions={29794f0ee13f249de2677b8de66b1e0b=testBlockEvictionOnGracefulStop1733306956936,,1733306956940.29794f0ee13f249de2677b8de66b1e0b., 1588230740=hbase:meta,,1.1588230740} 2024-12-04T10:09:19,880 INFO [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-04T10:09:19,880 DEBUG [RS:0;84486a41f81c:46309 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 29794f0ee13f249de2677b8de66b1e0b 2024-12-04T10:09:19,880 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T10:09:19,880 INFO [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T10:09:19,880 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T10:09:19,880 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T10:09:19,880 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T10:09:19,880 DEBUG [StoreFileCloser-29794f0ee13f249de2677b8de66b1e0b-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/default/testBlockEvictionOnGracefulStop1733306956936/29794f0ee13f249de2677b8de66b1e0b/family/945f1620af7d490989a9cdf442b53ea5 2024-12-04T10:09:19,880 INFO [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.96 KB heapSize=5.58 KB 2024-12-04T10:09:19,884 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/default/testBlockEvictionOnGracefulStop1733306956936/29794f0ee13f249de2677b8de66b1e0b/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-12-04T10:09:19,884 INFO [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733306956936,,1733306956940.29794f0ee13f249de2677b8de66b1e0b. 2024-12-04T10:09:19,884 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 29794f0ee13f249de2677b8de66b1e0b: Waiting for close lock at 1733306959879Running coprocessor pre-close hooks at 1733306959879Disabling compacts and flushes for region at 1733306959879Disabling writes for close at 1733306959879Writing region close event to WAL at 1733306959880 (+1 ms)Running coprocessor post-close hooks at 1733306959884 (+4 ms)Closed at 1733306959884 2024-12-04T10:09:19,885 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1733306956936,,1733306956940.29794f0ee13f249de2677b8de66b1e0b. 2024-12-04T10:09:19,889 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 12b6e502eeab408389b2203c37040a62 2024-12-04T10:09:19,891 INFO [regionserver/84486a41f81c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T10:09:19,899 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44233-0x101a11ad9b80003, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/84486a41f81c,44233,1733306959138 2024-12-04T10:09:19,899 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37329-0x101a11ad9b80000, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T10:09:19,899 INFO [RS:2;84486a41f81c:44233 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T10:09:19,899 ERROR [Time-limited test-EventThread {}] zookeeper.ClientCnxn$EventThread(581): Error while calling watcher. java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.trace.TraceUtil$$Lambda$249/0x00007f27f8477b20@28bcccd1 rejected from java.util.concurrent.ThreadPoolExecutor@6dbe47a2[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 3] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?] at java.util.concurrent.Executors$DelegatedExecutorService.execute(Executors.java:721) ~[?:?] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.process(ZKWatcher.java:613) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:579) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:554) ~[zookeeper-3.8.4.jar:3.8.4] 2024-12-04T10:09:19,903 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740/.tmp/info/12b6e502eeab408389b2203c37040a62 is 209, key is testBlockEvictionOnGracefulStop1733306956936,,1733306956940.29794f0ee13f249de2677b8de66b1e0b./info:regioninfo/1733306959258/Put/seqid=0 2024-12-04T10:09:19,904 INFO [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.74 KB at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740/.tmp/info/12b6e502eeab408389b2203c37040a62 2024-12-04T10:09:19,912 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 21b0304cf5404df4a8ddfdb7a4258090 2024-12-04T10:09:19,921 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [84486a41f81c,44233,1733306959138] 2024-12-04T10:09:19,923 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740/.tmp/ns/21b0304cf5404df4a8ddfdb7a4258090 is 43, key is default/ns:d/1733306956824/Put/seqid=0 2024-12-04T10:09:19,924 INFO [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740/.tmp/ns/21b0304cf5404df4a8ddfdb7a4258090 2024-12-04T10:09:19,929 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/84486a41f81c,44233,1733306959138 already deleted, retry=false 2024-12-04T10:09:19,929 INFO [regionserver/84486a41f81c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T10:09:19,929 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 84486a41f81c,44233,1733306959138 expired; onlineServers=1 2024-12-04T10:09:19,931 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 98d83ada8ed8484cb314c6b93f02d5d3 2024-12-04T10:09:19,942 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740/.tmp/table/98d83ada8ed8484cb314c6b93f02d5d3 is 80, key is testBlockEvictionOnGracefulStop1733306956936/table:state/1733306957323/Put/seqid=0 2024-12-04T10:09:19,942 INFO [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740/.tmp/table/98d83ada8ed8484cb314c6b93f02d5d3 2024-12-04T10:09:19,944 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740/.tmp/info/12b6e502eeab408389b2203c37040a62 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740/info/12b6e502eeab408389b2203c37040a62 2024-12-04T10:09:19,945 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740/info/12b6e502eeab408389b2203c37040a62, delay=921 ms 2024-12-04T10:09:19,947 INFO [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740/info/12b6e502eeab408389b2203c37040a62, entries=14, sequenceid=13, filesize=7.5 K 2024-12-04T10:09:19,947 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740/.tmp/ns/21b0304cf5404df4a8ddfdb7a4258090 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740/ns/21b0304cf5404df4a8ddfdb7a4258090 2024-12-04T10:09:19,948 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740/ns/21b0304cf5404df4a8ddfdb7a4258090, delay=956 ms 2024-12-04T10:09:19,949 INFO [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740/ns/21b0304cf5404df4a8ddfdb7a4258090, entries=2, sequenceid=13, filesize=5.0 K 2024-12-04T10:09:19,949 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740/.tmp/table/98d83ada8ed8484cb314c6b93f02d5d3 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740/table/98d83ada8ed8484cb314c6b93f02d5d3 2024-12-04T10:09:19,950 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740/table/98d83ada8ed8484cb314c6b93f02d5d3, delay=922 ms 2024-12-04T10:09:19,951 INFO [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740/table/98d83ada8ed8484cb314c6b93f02d5d3, entries=2, sequenceid=13, filesize=5.3 K 2024-12-04T10:09:19,952 INFO [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 71ms, sequenceid=13, compaction requested=false 2024-12-04T10:09:19,952 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740/info/12b6e502eeab408389b2203c37040a62 2024-12-04T10:09:19,953 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740/ns/21b0304cf5404df4a8ddfdb7a4258090 2024-12-04T10:09:19,953 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740/table/98d83ada8ed8484cb314c6b93f02d5d3 2024-12-04T10:09:19,957 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/data/hbase/meta/1588230740/recovered.edits/16.seqid, newMaxSeqId=16, maxSeqId=1 2024-12-04T10:09:19,957 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T10:09:19,957 INFO [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T10:09:19,957 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733306959880Running coprocessor pre-close hooks at 1733306959880Disabling compacts and flushes for region at 1733306959880Disabling writes for close at 1733306959880Obtaining lock to block concurrent updates at 1733306959880Preparing flush snapshotting stores in 1588230740 at 1733306959880Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3035, getHeapSize=5648, getOffHeapSize=0, getCellsCount=22 at 1733306959881 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733306959881Flushing 1588230740/info: creating writer at 1733306959881Flushing 1588230740/info: appending metadata at 1733306959903 (+22 ms)Flushing 1588230740/info: closing flushed file at 1733306959903Flushing 1588230740/ns: creating writer at 1733306959907 (+4 ms)Flushing 1588230740/ns: appending metadata at 1733306959923 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1733306959923Flushing 1588230740/table: creating writer at 1733306959925 (+2 ms)Flushing 1588230740/table: appending metadata at 1733306959941 (+16 ms)Flushing 1588230740/table: closing flushed file at 1733306959941Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7821fd3e: reopening flushed file at 1733306959944 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74abf7c0: reopening flushed file at 1733306959947 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f68661b: reopening flushed file at 1733306959949 (+2 ms)Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 71ms, sequenceid=13, compaction requested=false at 1733306959952 (+3 ms)Writing region close event to WAL at 1733306959953 (+1 ms)Running coprocessor post-close hooks at 1733306959957 (+4 ms)Closed at 1733306959957 2024-12-04T10:09:19,958 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-04T10:09:20,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44233-0x101a11ad9b80003, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T10:09:20,021 INFO [RS:2;84486a41f81c:44233 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T10:09:20,022 INFO [RS:2;84486a41f81c:44233 {}] regionserver.HRegionServer(1031): Exiting; stopping=84486a41f81c,44233,1733306959138; zookeeper connection closed. 2024-12-04T10:09:20,022 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44233-0x101a11ad9b80003, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T10:09:20,022 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@317b89f2 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@317b89f2 2024-12-04T10:09:20,062 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-04T10:09:20,063 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-04T10:09:20,080 INFO [RS:0;84486a41f81c:46309 {}] regionserver.HRegionServer(976): stopping server 84486a41f81c,46309,1733306955769; all regions closed. 2024-12-04T10:09:20,083 DEBUG [RS:0;84486a41f81c:46309 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/oldWALs 2024-12-04T10:09:20,083 INFO [RS:0;84486a41f81c:46309 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 84486a41f81c%2C46309%2C1733306955769.meta:.meta(num 1733306956772) 2024-12-04T10:09:20,084 DEBUG [RS:0;84486a41f81c:46309 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/oldWALs 2024-12-04T10:09:20,085 INFO [RS:0;84486a41f81c:46309 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 84486a41f81c%2C46309%2C1733306955769:(num 1733306956459) 2024-12-04T10:09:20,085 DEBUG [RS:0;84486a41f81c:46309 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T10:09:20,085 INFO [RS:0;84486a41f81c:46309 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T10:09:20,085 INFO [RS:0;84486a41f81c:46309 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T10:09:20,085 INFO [RS:0;84486a41f81c:46309 {}] hbase.ChoreService(370): Chore service for: regionserver/84486a41f81c:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-04T10:09:20,085 INFO [RS:0;84486a41f81c:46309 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T10:09:20,085 INFO [regionserver/84486a41f81c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T10:09:20,085 INFO [RS:0;84486a41f81c:46309 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46309 2024-12-04T10:09:20,096 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46309-0x101a11ad9b80001, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/84486a41f81c,46309,1733306955769 2024-12-04T10:09:20,096 INFO [RS:0;84486a41f81c:46309 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T10:09:20,096 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37329-0x101a11ad9b80000, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T10:09:20,105 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [84486a41f81c,46309,1733306955769] 2024-12-04T10:09:20,113 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/84486a41f81c,46309,1733306955769 already deleted, retry=false 2024-12-04T10:09:20,113 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 84486a41f81c,46309,1733306955769 expired; onlineServers=0 2024-12-04T10:09:20,113 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '84486a41f81c,37329,1733306955606' ***** 2024-12-04T10:09:20,113 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-04T10:09:20,113 INFO [M:0;84486a41f81c:37329 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T10:09:20,113 INFO [M:0;84486a41f81c:37329 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T10:09:20,114 DEBUG [M:0;84486a41f81c:37329 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-04T10:09:20,114 DEBUG [M:0;84486a41f81c:37329 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-04T10:09:20,114 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-04T10:09:20,114 DEBUG [master/84486a41f81c:0:becomeActiveMaster-HFileCleaner.large.0-1733306956139 {}] cleaner.HFileCleaner(306): Exit Thread[master/84486a41f81c:0:becomeActiveMaster-HFileCleaner.large.0-1733306956139,5,FailOnTimeoutGroup] 2024-12-04T10:09:20,114 DEBUG [master/84486a41f81c:0:becomeActiveMaster-HFileCleaner.small.0-1733306956140 {}] cleaner.HFileCleaner(306): Exit Thread[master/84486a41f81c:0:becomeActiveMaster-HFileCleaner.small.0-1733306956140,5,FailOnTimeoutGroup] 2024-12-04T10:09:20,114 INFO [M:0;84486a41f81c:37329 {}] hbase.ChoreService(370): Chore service for: master/84486a41f81c:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-04T10:09:20,115 INFO [M:0;84486a41f81c:37329 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T10:09:20,115 DEBUG [M:0;84486a41f81c:37329 {}] master.HMaster(1795): Stopping service threads 2024-12-04T10:09:20,115 INFO [M:0;84486a41f81c:37329 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-04T10:09:20,115 INFO [M:0;84486a41f81c:37329 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T10:09:20,115 INFO [M:0;84486a41f81c:37329 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-04T10:09:20,116 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-04T10:09:20,121 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37329-0x101a11ad9b80000, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-04T10:09:20,121 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37329-0x101a11ad9b80000, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T10:09:20,122 DEBUG [M:0;84486a41f81c:37329 {}] zookeeper.ZKUtil(347): master:37329-0x101a11ad9b80000, quorum=127.0.0.1:57330, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-04T10:09:20,122 WARN [M:0;84486a41f81c:37329 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-04T10:09:20,122 INFO [M:0;84486a41f81c:37329 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/.lastflushedseqids 2024-12-04T10:09:20,129 INFO [M:0;84486a41f81c:37329 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-04T10:09:20,129 INFO [M:0;84486a41f81c:37329 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-04T10:09:20,129 DEBUG [M:0;84486a41f81c:37329 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T10:09:20,129 INFO [M:0;84486a41f81c:37329 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T10:09:20,129 DEBUG [M:0;84486a41f81c:37329 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T10:09:20,129 DEBUG [M:0;84486a41f81c:37329 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T10:09:20,129 DEBUG [M:0;84486a41f81c:37329 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T10:09:20,129 INFO [M:0;84486a41f81c:37329 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=32.84 KB heapSize=41.09 KB 2024-12-04T10:09:20,137 DEBUG [M:0;84486a41f81c:37329 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 7ad95b166c614e19b504819362e93340 2024-12-04T10:09:20,149 DEBUG [M:0;84486a41f81c:37329 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7ad95b166c614e19b504819362e93340 is 82, key is hbase:meta,,1/info:regioninfo/1733306956799/Put/seqid=0 2024-12-04T10:09:20,150 INFO [M:0;84486a41f81c:37329 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7ad95b166c614e19b504819362e93340 2024-12-04T10:09:20,157 DEBUG [M:0;84486a41f81c:37329 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 176b341186064910a5ee5d1ae33ae1a2 2024-12-04T10:09:20,171 DEBUG [M:0;84486a41f81c:37329 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/176b341186064910a5ee5d1ae33ae1a2 is 783, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733306957328/Put/seqid=0 2024-12-04T10:09:20,171 INFO [M:0;84486a41f81c:37329 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.10 KB at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/176b341186064910a5ee5d1ae33ae1a2 2024-12-04T10:09:20,178 DEBUG [M:0;84486a41f81c:37329 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 4579914f43194e7c978faaa6a6f556c7 2024-12-04T10:09:20,189 DEBUG [M:0;84486a41f81c:37329 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4579914f43194e7c978faaa6a6f556c7 is 69, key is 84486a41f81c,44233,1733306959138/rs:state/1733306959271/Put/seqid=0 2024-12-04T10:09:20,189 INFO [M:0;84486a41f81c:37329 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4579914f43194e7c978faaa6a6f556c7 2024-12-04T10:09:20,191 INFO [M:0;84486a41f81c:37329 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4579914f43194e7c978faaa6a6f556c7 2024-12-04T10:09:20,191 DEBUG [M:0;84486a41f81c:37329 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7ad95b166c614e19b504819362e93340 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7ad95b166c614e19b504819362e93340 2024-12-04T10:09:20,193 INFO [M:0;84486a41f81c:37329 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7ad95b166c614e19b504819362e93340, entries=8, sequenceid=83, filesize=5.5 K 2024-12-04T10:09:20,193 DEBUG [M:0;84486a41f81c:37329 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/176b341186064910a5ee5d1ae33ae1a2 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/176b341186064910a5ee5d1ae33ae1a2 2024-12-04T10:09:20,195 INFO [M:0;84486a41f81c:37329 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/176b341186064910a5ee5d1ae33ae1a2, entries=9, sequenceid=83, filesize=6.4 K 2024-12-04T10:09:20,195 DEBUG [M:0;84486a41f81c:37329 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4579914f43194e7c978faaa6a6f556c7 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4579914f43194e7c978faaa6a6f556c7 2024-12-04T10:09:20,196 INFO [M:0;84486a41f81c:37329 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4579914f43194e7c978faaa6a6f556c7 2024-12-04T10:09:20,197 INFO [M:0;84486a41f81c:37329 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c6605a88-5557-07a7-510d-7d2e809c3966/aa34762f-8cdd-bb70-f886-086ec89d31ee/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4579914f43194e7c978faaa6a6f556c7, entries=3, sequenceid=83, filesize=5.3 K 2024-12-04T10:09:20,197 INFO [M:0;84486a41f81c:37329 {}] regionserver.HRegion(3140): Finished flush of dataSize ~32.84 KB/33626, heapSize ~40.79 KB/41768, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 68ms, sequenceid=83, compaction requested=false 2024-12-04T10:09:20,199 INFO [M:0;84486a41f81c:37329 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T10:09:20,199 DEBUG [M:0;84486a41f81c:37329 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733306960129Disabling compacts and flushes for region at 1733306960129Disabling writes for close at 1733306960129Obtaining lock to block concurrent updates at 1733306960129Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733306960129Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=33626, getHeapSize=42008, getOffHeapSize=0, getCellsCount=98 at 1733306960130 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733306960130Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733306960130Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733306960148 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733306960149 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733306960151 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733306960170 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733306960170Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733306960173 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733306960188 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733306960188Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6405ca31: reopening flushed file at 1733306960191 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d697cba: reopening flushed file at 1733306960193 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e99bcaa: reopening flushed file at 1733306960195 (+2 ms)Finished flush of dataSize ~32.84 KB/33626, heapSize ~40.79 KB/41768, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 68ms, sequenceid=83, compaction requested=false at 1733306960197 (+2 ms)Writing region close event to WAL at 1733306960198 (+1 ms)Closed at 1733306960198 2024-12-04T10:09:20,200 INFO [M:0;84486a41f81c:37329 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-04T10:09:20,200 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T10:09:20,200 INFO [M:0;84486a41f81c:37329 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37329 2024-12-04T10:09:20,200 INFO [M:0;84486a41f81c:37329 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T10:09:20,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46309-0x101a11ad9b80001, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T10:09:20,205 INFO [RS:0;84486a41f81c:46309 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T10:09:20,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46309-0x101a11ad9b80001, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T10:09:20,205 INFO [RS:0;84486a41f81c:46309 {}] regionserver.HRegionServer(1031): Exiting; stopping=84486a41f81c,46309,1733306955769; zookeeper connection closed. 2024-12-04T10:09:20,205 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6d922bfe {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6d922bfe 2024-12-04T10:09:20,205 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-04T10:09:20,336 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37329-0x101a11ad9b80000, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T10:09:20,336 INFO [M:0;84486a41f81c:37329 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T10:09:20,336 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37329-0x101a11ad9b80000, quorum=127.0.0.1:57330, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T10:09:20,343 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-04T10:09:20,345 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-04T10:09:20,345 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T10:09:20,353 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=71 (was 50) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=787 (was 830), ProcessCount=11 (was 11), AvailableMemoryMB=10565 (was 10639)