2024-11-07 11:47:59,021 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-07 11:47:59,033 main DEBUG Took 0.010152 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-07 11:47:59,034 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-07 11:47:59,034 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-07 11:47:59,035 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-07 11:47:59,036 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 11:47:59,044 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-07 11:47:59,065 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 11:47:59,066 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 11:47:59,067 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 11:47:59,068 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 11:47:59,068 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 11:47:59,069 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 11:47:59,070 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 11:47:59,071 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 11:47:59,071 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 11:47:59,072 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 11:47:59,073 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 11:47:59,073 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 11:47:59,074 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 11:47:59,075 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 11:47:59,075 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 11:47:59,076 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 11:47:59,076 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 11:47:59,077 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 11:47:59,078 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 11:47:59,078 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 11:47:59,079 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 11:47:59,079 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 11:47:59,080 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 11:47:59,080 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 11:47:59,081 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 11:47:59,081 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-07 11:47:59,083 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 11:47:59,085 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-07 11:47:59,087 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-07 11:47:59,088 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-07 11:47:59,089 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-07 11:47:59,090 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-07 11:47:59,102 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-07 11:47:59,106 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-07 11:47:59,108 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-07 11:47:59,109 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-07 11:47:59,109 main DEBUG createAppenders(={Console}) 2024-11-07 11:47:59,110 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-07 11:47:59,110 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-07 11:47:59,111 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-07 11:47:59,112 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-07 11:47:59,112 main DEBUG OutputStream closed 2024-11-07 11:47:59,112 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-07 11:47:59,113 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-07 11:47:59,113 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-07 11:47:59,204 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-07 11:47:59,207 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-07 11:47:59,208 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-07 11:47:59,209 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-07 11:47:59,210 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-07 11:47:59,210 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-07 11:47:59,211 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-07 11:47:59,211 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-07 11:47:59,212 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-07 11:47:59,212 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-07 11:47:59,213 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-07 11:47:59,213 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-07 11:47:59,214 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-07 11:47:59,214 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-07 11:47:59,214 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-07 11:47:59,215 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-07 11:47:59,215 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-07 11:47:59,216 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-07 11:47:59,218 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-07 11:47:59,218 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-07 11:47:59,219 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-07 11:47:59,219 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-07T11:47:59,236 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-07 11:47:59,239 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-07 11:47:59,239 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-07T11:47:59,529 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e 2024-11-07T11:47:59,556 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=2206, ProcessCount=11, AvailableMemoryMB=3279 2024-11-07T11:47:59,659 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-07T11:47:59,757 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T11:47:59,773 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/cluster_e2c46c74-e328-1c89-de5c-ad23312e3745, deleteOnExit=true 2024-11-07T11:47:59,879 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/cluster_e2c46c74-e328-1c89-de5c-ad23312e3745/zookeeper_0, clientPort=64253, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/cluster_e2c46c74-e328-1c89-de5c-ad23312e3745/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/cluster_e2c46c74-e328-1c89-de5c-ad23312e3745/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-07T11:47:59,891 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=64253 2024-11-07T11:47:59,892 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T11:47:59,993 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47 with version=8 2024-11-07T11:47:59,993 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/hbase-staging 2024-11-07T11:48:00,096 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-07T11:48:00,379 INFO [Time-limited test {}] client.ConnectionUtils(128): master/f30636bb2128:0 server-side Connection retries=45 2024-11-07T11:48:00,391 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T11:48:00,392 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-07T11:48:00,400 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-07T11:48:00,400 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T11:48:00,400 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-07T11:48:00,544 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-07T11:48:00,642 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T11:48:00,679 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-07T11:48:00,689 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-07T11:48:00,694 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-07T11:48:00,730 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 17858 (auto-detected) 2024-11-07T11:48:00,732 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-07T11:48:00,752 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35319 2024-11-07T11:48:00,784 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35319 connecting to ZooKeeper ensemble=127.0.0.1:64253 2024-11-07T11:48:00,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:353190x0, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-07T11:48:00,824 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35319-0x1001a11f38e0000 connected 2024-11-07T11:48:00,850 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T11:48:00,851 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T11:48:00,859 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35319-0x1001a11f38e0000, quorum=127.0.0.1:64253, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T11:48:00,863 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47, hbase.cluster.distributed=false 2024-11-07T11:48:00,889 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35319-0x1001a11f38e0000, quorum=127.0.0.1:64253, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-07T11:48:00,896 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35319 2024-11-07T11:48:00,897 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35319 2024-11-07T11:48:00,898 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35319 2024-11-07T11:48:00,901 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35319 2024-11-07T11:48:00,902 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35319 2024-11-07T11:48:00,992 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f30636bb2128:0 server-side Connection retries=45 2024-11-07T11:48:00,994 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T11:48:00,995 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-07T11:48:00,995 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-07T11:48:00,995 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T11:48:00,995 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-07T11:48:00,998 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-07T11:48:01,001 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-07T11:48:01,002 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38071 2024-11-07T11:48:01,004 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38071 connecting to ZooKeeper ensemble=127.0.0.1:64253 2024-11-07T11:48:01,004 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T11:48:01,005 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T11:48:01,009 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:380710x0, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-07T11:48:01,011 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38071-0x1001a11f38e0001 connected 2024-11-07T11:48:01,011 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38071-0x1001a11f38e0001, quorum=127.0.0.1:64253, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T11:48:01,017 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-07T11:48:01,031 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/bucket.cache 2024-11-07T11:48:01,037 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-07T11:48:01,039 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-07T11:48:01,041 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-07T11:48:01,042 INFO [Thread-10 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-07T11:48:01,043 WARN [Thread-10 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-07T11:48:01,050 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-07T11:48:01,054 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-07T11:48:01,054 INFO [Thread-10 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/bucket.persistence is complete. 2024-11-07T11:48:01,057 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-07T11:48:01,060 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38071-0x1001a11f38e0001, quorum=127.0.0.1:64253, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-07T11:48:01,067 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38071-0x1001a11f38e0001, quorum=127.0.0.1:64253, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-07T11:48:01,068 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38071 2024-11-07T11:48:01,068 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38071 2024-11-07T11:48:01,073 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38071 2024-11-07T11:48:01,075 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38071 2024-11-07T11:48:01,076 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38071 2024-11-07T11:48:01,104 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f30636bb2128:0 server-side Connection retries=45 2024-11-07T11:48:01,105 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T11:48:01,105 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-07T11:48:01,106 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-07T11:48:01,106 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T11:48:01,106 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-07T11:48:01,106 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-07T11:48:01,107 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-07T11:48:01,108 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40315 2024-11-07T11:48:01,111 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40315 connecting to ZooKeeper ensemble=127.0.0.1:64253 2024-11-07T11:48:01,111 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T11:48:01,113 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T11:48:01,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:403150x0, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-07T11:48:01,119 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40315-0x1001a11f38e0002 connected 2024-11-07T11:48:01,119 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40315-0x1001a11f38e0002, quorum=127.0.0.1:64253, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T11:48:01,120 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-07T11:48:01,121 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/bucket.cache 2024-11-07T11:48:01,121 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-07T11:48:01,122 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-07T11:48:01,122 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-07T11:48:01,125 INFO [Thread-11 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-07T11:48:01,125 WARN [Thread-11 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-07T11:48:01,126 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-07T11:48:01,126 INFO [Thread-11 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/bucket.persistence is complete. 2024-11-07T11:48:01,127 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-07T11:48:01,128 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-07T11:48:01,130 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40315-0x1001a11f38e0002, quorum=127.0.0.1:64253, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-07T11:48:01,132 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40315-0x1001a11f38e0002, quorum=127.0.0.1:64253, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-07T11:48:01,136 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40315 2024-11-07T11:48:01,137 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40315 2024-11-07T11:48:01,137 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40315 2024-11-07T11:48:01,138 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40315 2024-11-07T11:48:01,140 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40315 2024-11-07T11:48:01,157 DEBUG [M:0;f30636bb2128:35319 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;f30636bb2128:35319 2024-11-07T11:48:01,159 INFO [master/f30636bb2128:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/f30636bb2128,35319,1730980080176 2024-11-07T11:48:01,167 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40315-0x1001a11f38e0002, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T11:48:01,167 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38071-0x1001a11f38e0001, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T11:48:01,167 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35319-0x1001a11f38e0000, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T11:48:01,170 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35319-0x1001a11f38e0000, quorum=127.0.0.1:64253, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/f30636bb2128,35319,1730980080176 2024-11-07T11:48:01,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38071-0x1001a11f38e0001, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-07T11:48:01,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40315-0x1001a11f38e0002, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-07T11:48:01,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35319-0x1001a11f38e0000, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:48:01,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40315-0x1001a11f38e0002, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:48:01,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38071-0x1001a11f38e0001, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:48:01,195 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35319-0x1001a11f38e0000, quorum=127.0.0.1:64253, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-07T11:48:01,197 INFO [master/f30636bb2128:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/f30636bb2128,35319,1730980080176 from backup master directory 2024-11-07T11:48:01,200 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40315-0x1001a11f38e0002, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T11:48:01,200 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38071-0x1001a11f38e0001, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T11:48:01,200 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35319-0x1001a11f38e0000, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/f30636bb2128,35319,1730980080176 2024-11-07T11:48:01,200 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35319-0x1001a11f38e0000, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T11:48:01,201 WARN [master/f30636bb2128:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-07T11:48:01,201 INFO [master/f30636bb2128:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=f30636bb2128,35319,1730980080176 2024-11-07T11:48:01,204 INFO [master/f30636bb2128:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-07T11:48:01,205 INFO [master/f30636bb2128:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-07T11:48:01,213 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/hbase.id] with ID: 69e30f5c-659f-49ac-a3c3-809bd52e6096 2024-11-07T11:48:01,213 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/.tmp/hbase.id 2024-11-07T11:48:01,220 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/hbase.id] 2024-11-07T11:48:01,261 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T11:48:01,266 INFO [master/f30636bb2128:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-07T11:48:01,286 INFO [master/f30636bb2128:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-07T11:48:01,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38071-0x1001a11f38e0001, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:48:01,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40315-0x1001a11f38e0002, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:48:01,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35319-0x1001a11f38e0000, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:48:01,317 INFO [master/f30636bb2128:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T11:48:01,320 INFO [master/f30636bb2128:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-07T11:48:01,367 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T11:48:01,373 INFO [master/f30636bb2128:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T11:48:01,431 INFO [master/f30636bb2128:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/MasterData/data/master/store 2024-11-07T11:48:01,452 INFO [master/f30636bb2128:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-07T11:48:01,455 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:48:01,457 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-07T11:48:01,457 INFO [master/f30636bb2128:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T11:48:01,457 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T11:48:01,459 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-07T11:48:01,459 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T11:48:01,459 INFO [master/f30636bb2128:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T11:48:01,459 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1730980081457Disabling compacts and flushes for region at 1730980081457Disabling writes for close at 1730980081459 (+2 ms)Writing region close event to WAL at 1730980081459Closed at 1730980081459 2024-11-07T11:48:01,460 WARN [master/f30636bb2128:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/MasterData/data/master/store/.initializing 2024-11-07T11:48:01,460 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/MasterData/WALs/f30636bb2128,35319,1730980080176 2024-11-07T11:48:01,473 INFO [master/f30636bb2128:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-07T11:48:01,485 INFO [master/f30636bb2128:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f30636bb2128%2C35319%2C1730980080176, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/MasterData/WALs/f30636bb2128,35319,1730980080176, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/MasterData/oldWALs, maxLogs=10 2024-11-07T11:48:01,526 INFO [master/f30636bb2128:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/MasterData/WALs/f30636bb2128,35319,1730980080176/f30636bb2128%2C35319%2C1730980080176.1730980081489 2024-11-07T11:48:01,527 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-07T11:48:01,528 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-07T11:48:01,528 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:48:01,533 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T11:48:01,534 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T11:48:01,581 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T11:48:01,609 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-07T11:48:01,613 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:48:01,615 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T11:48:01,616 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T11:48:01,620 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-07T11:48:01,620 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:48:01,621 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T11:48:01,621 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T11:48:01,625 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-07T11:48:01,626 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:48:01,626 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T11:48:01,626 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T11:48:01,630 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-07T11:48:01,631 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:48:01,631 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T11:48:01,631 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T11:48:01,636 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-07T11:48:01,637 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-07T11:48:01,641 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T11:48:01,642 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T11:48:01,645 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-07T11:48:01,648 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T11:48:01,656 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T11:48:01,657 INFO [master/f30636bb2128:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62810366, jitterRate=-0.06405261158943176}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-07T11:48:01,663 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1730980081551Initializing all the Stores at 1730980081552 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730980081552Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730980081554 (+2 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730980081554Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730980081554Cleaning up temporary data from old regions at 1730980081642 (+88 ms)Region opened successfully at 1730980081663 (+21 ms) 2024-11-07T11:48:01,665 INFO [master/f30636bb2128:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-07T11:48:01,699 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17da0401, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f30636bb2128/172.17.0.2:0 2024-11-07T11:48:01,742 INFO [master/f30636bb2128:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-07T11:48:01,755 INFO [master/f30636bb2128:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-07T11:48:01,755 INFO [master/f30636bb2128:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-07T11:48:01,758 INFO [master/f30636bb2128:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T11:48:01,759 INFO [master/f30636bb2128:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T11:48:01,766 INFO [master/f30636bb2128:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-07T11:48:01,766 INFO [master/f30636bb2128:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-07T11:48:01,796 INFO [master/f30636bb2128:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-07T11:48:01,808 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35319-0x1001a11f38e0000, quorum=127.0.0.1:64253, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-07T11:48:01,811 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-07T11:48:01,814 INFO [master/f30636bb2128:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-07T11:48:01,816 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35319-0x1001a11f38e0000, quorum=127.0.0.1:64253, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-07T11:48:01,818 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-07T11:48:01,821 INFO [master/f30636bb2128:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-07T11:48:01,825 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35319-0x1001a11f38e0000, quorum=127.0.0.1:64253, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-07T11:48:01,827 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-07T11:48:01,829 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35319-0x1001a11f38e0000, quorum=127.0.0.1:64253, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-07T11:48:01,831 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-07T11:48:01,851 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35319-0x1001a11f38e0000, quorum=127.0.0.1:64253, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-07T11:48:01,853 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-07T11:48:01,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35319-0x1001a11f38e0000, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-07T11:48:01,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38071-0x1001a11f38e0001, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-07T11:48:01,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40315-0x1001a11f38e0002, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-07T11:48:01,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35319-0x1001a11f38e0000, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:48:01,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38071-0x1001a11f38e0001, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:48:01,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40315-0x1001a11f38e0002, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:48:01,873 INFO [master/f30636bb2128:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=f30636bb2128,35319,1730980080176, sessionid=0x1001a11f38e0000, setting cluster-up flag (Was=false) 2024-11-07T11:48:01,890 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35319-0x1001a11f38e0000, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:48:01,890 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40315-0x1001a11f38e0002, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:48:01,890 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38071-0x1001a11f38e0001, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:48:01,896 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-07T11:48:01,898 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f30636bb2128,35319,1730980080176 2024-11-07T11:48:01,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35319-0x1001a11f38e0000, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:48:01,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38071-0x1001a11f38e0001, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:48:01,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40315-0x1001a11f38e0002, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:48:01,912 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-07T11:48:01,914 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f30636bb2128,35319,1730980080176 2024-11-07T11:48:01,919 INFO [master/f30636bb2128:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-07T11:48:01,946 INFO [RS:1;f30636bb2128:40315 {}] regionserver.HRegionServer(746): ClusterId : 69e30f5c-659f-49ac-a3c3-809bd52e6096 2024-11-07T11:48:01,946 INFO [RS:0;f30636bb2128:38071 {}] regionserver.HRegionServer(746): ClusterId : 69e30f5c-659f-49ac-a3c3-809bd52e6096 2024-11-07T11:48:01,949 DEBUG [RS:0;f30636bb2128:38071 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-07T11:48:01,949 DEBUG [RS:1;f30636bb2128:40315 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-07T11:48:01,955 DEBUG [RS:0;f30636bb2128:38071 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-07T11:48:01,956 DEBUG [RS:1;f30636bb2128:40315 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-07T11:48:01,956 DEBUG [RS:0;f30636bb2128:38071 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-07T11:48:01,956 DEBUG [RS:1;f30636bb2128:40315 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-07T11:48:01,959 DEBUG [RS:0;f30636bb2128:38071 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-07T11:48:01,960 DEBUG [RS:1;f30636bb2128:40315 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-07T11:48:01,960 DEBUG [RS:0;f30636bb2128:38071 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24bb5a69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f30636bb2128/172.17.0.2:0 2024-11-07T11:48:01,961 DEBUG [RS:1;f30636bb2128:40315 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77672fe4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f30636bb2128/172.17.0.2:0 2024-11-07T11:48:01,980 DEBUG [RS:0;f30636bb2128:38071 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;f30636bb2128:38071 2024-11-07T11:48:01,981 DEBUG [RS:1;f30636bb2128:40315 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;f30636bb2128:40315 2024-11-07T11:48:01,983 INFO [RS:0;f30636bb2128:38071 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-07T11:48:01,983 INFO [RS:1;f30636bb2128:40315 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-07T11:48:01,983 INFO [RS:0;f30636bb2128:38071 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-07T11:48:01,983 INFO [RS:1;f30636bb2128:40315 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-07T11:48:01,983 DEBUG [RS:0;f30636bb2128:38071 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-07T11:48:01,983 DEBUG [RS:1;f30636bb2128:40315 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-07T11:48:01,987 INFO [RS:1;f30636bb2128:40315 {}] regionserver.HRegionServer(2659): reportForDuty to master=f30636bb2128,35319,1730980080176 with port=40315, startcode=1730980081104 2024-11-07T11:48:01,987 INFO [RS:0;f30636bb2128:38071 {}] regionserver.HRegionServer(2659): reportForDuty to master=f30636bb2128,35319,1730980080176 with port=38071, startcode=1730980080953 2024-11-07T11:48:02,002 DEBUG [RS:1;f30636bb2128:40315 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-07T11:48:02,002 DEBUG [RS:0;f30636bb2128:38071 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-07T11:48:02,009 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T11:48:02,021 INFO [master/f30636bb2128:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-07T11:48:02,029 INFO [master/f30636bb2128:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-07T11:48:02,035 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: f30636bb2128,35319,1730980080176 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-07T11:48:02,044 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/f30636bb2128:0, corePoolSize=5, maxPoolSize=5 2024-11-07T11:48:02,044 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/f30636bb2128:0, corePoolSize=5, maxPoolSize=5 2024-11-07T11:48:02,044 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/f30636bb2128:0, corePoolSize=5, maxPoolSize=5 2024-11-07T11:48:02,045 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/f30636bb2128:0, corePoolSize=5, maxPoolSize=5 2024-11-07T11:48:02,045 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/f30636bb2128:0, corePoolSize=10, maxPoolSize=10 2024-11-07T11:48:02,045 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:02,046 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/f30636bb2128:0, corePoolSize=2, maxPoolSize=2 2024-11-07T11:48:02,046 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:02,060 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-07T11:48:02,061 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-07T11:48:02,066 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:48:02,066 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-07T11:48:02,069 INFO [master/f30636bb2128:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1730980112069 2024-11-07T11:48:02,070 INFO [master/f30636bb2128:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-07T11:48:02,071 INFO [master/f30636bb2128:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-07T11:48:02,076 INFO [master/f30636bb2128:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-07T11:48:02,076 INFO [master/f30636bb2128:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-07T11:48:02,077 INFO [master/f30636bb2128:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-07T11:48:02,077 INFO [master/f30636bb2128:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-07T11:48:02,078 INFO [master/f30636bb2128:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:02,085 INFO [master/f30636bb2128:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-07T11:48:02,087 INFO [master/f30636bb2128:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-07T11:48:02,088 INFO [master/f30636bb2128:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-07T11:48:02,092 INFO [master/f30636bb2128:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-07T11:48:02,092 INFO [master/f30636bb2128:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-07T11:48:02,092 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55715, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-07T11:48:02,093 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56541, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-07T11:48:02,094 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/f30636bb2128:0:becomeActiveMaster-HFileCleaner.large.0-1730980082094,5,FailOnTimeoutGroup] 2024-11-07T11:48:02,095 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/f30636bb2128:0:becomeActiveMaster-HFileCleaner.small.0-1730980082095,5,FailOnTimeoutGroup] 2024-11-07T11:48:02,095 INFO [master/f30636bb2128:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:02,095 INFO [master/f30636bb2128:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-07T11:48:02,097 INFO [master/f30636bb2128:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:02,097 INFO [master/f30636bb2128:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:02,100 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35319 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f30636bb2128,38071,1730980080953 2024-11-07T11:48:02,103 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35319 {}] master.ServerManager(517): Registering regionserver=f30636bb2128,38071,1730980080953 2024-11-07T11:48:02,105 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-07T11:48:02,105 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47 2024-11-07T11:48:02,118 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35319 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f30636bb2128,40315,1730980081104 2024-11-07T11:48:02,118 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35319 {}] master.ServerManager(517): Registering regionserver=f30636bb2128,40315,1730980081104 2024-11-07T11:48:02,123 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:48:02,123 DEBUG [RS:0;f30636bb2128:38071 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47 2024-11-07T11:48:02,123 DEBUG [RS:1;f30636bb2128:40315 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47 2024-11-07T11:48:02,123 DEBUG [RS:0;f30636bb2128:38071 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-07T11:48:02,123 DEBUG [RS:1;f30636bb2128:40315 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-07T11:48:02,123 DEBUG [RS:0;f30636bb2128:38071 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-07T11:48:02,123 DEBUG [RS:1;f30636bb2128:40315 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-07T11:48:02,124 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-07T11:48:02,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35319-0x1001a11f38e0000, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-07T11:48:02,134 DEBUG [RS:1;f30636bb2128:40315 {}] zookeeper.ZKUtil(111): regionserver:40315-0x1001a11f38e0002, quorum=127.0.0.1:64253, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f30636bb2128,40315,1730980081104 2024-11-07T11:48:02,134 WARN [RS:1;f30636bb2128:40315 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-07T11:48:02,134 INFO [RS:1;f30636bb2128:40315 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T11:48:02,134 DEBUG [RS:1;f30636bb2128:40315 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/WALs/f30636bb2128,40315,1730980081104 2024-11-07T11:48:02,135 DEBUG [RS:0;f30636bb2128:38071 {}] zookeeper.ZKUtil(111): regionserver:38071-0x1001a11f38e0001, quorum=127.0.0.1:64253, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f30636bb2128,38071,1730980080953 2024-11-07T11:48:02,135 WARN [RS:0;f30636bb2128:38071 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-07T11:48:02,136 INFO [RS:0;f30636bb2128:38071 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T11:48:02,136 DEBUG [RS:0;f30636bb2128:38071 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/WALs/f30636bb2128,38071,1730980080953 2024-11-07T11:48:02,140 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f30636bb2128,40315,1730980081104] 2024-11-07T11:48:02,140 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f30636bb2128,38071,1730980080953] 2024-11-07T11:48:02,140 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-07T11:48:02,141 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:48:02,141 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T11:48:02,141 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-07T11:48:02,150 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-07T11:48:02,150 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:48:02,150 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T11:48:02,151 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-07T11:48:02,165 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-07T11:48:02,166 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:48:02,166 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T11:48:02,166 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-07T11:48:02,168 INFO [RS:0;f30636bb2128:38071 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-07T11:48:02,168 INFO [RS:1;f30636bb2128:40315 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-07T11:48:02,178 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-07T11:48:02,178 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:48:02,178 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T11:48:02,179 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-07T11:48:02,180 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740 2024-11-07T11:48:02,180 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740 2024-11-07T11:48:02,181 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-07T11:48:02,181 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-07T11:48:02,181 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-07T11:48:02,183 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-07T11:48:02,186 INFO [RS:1;f30636bb2128:40315 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-07T11:48:02,186 INFO [RS:0;f30636bb2128:38071 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-07T11:48:02,192 INFO [RS:1;f30636bb2128:40315 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-07T11:48:02,193 INFO [RS:1;f30636bb2128:40315 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:02,193 INFO [RS:0;f30636bb2128:38071 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-07T11:48:02,193 INFO [RS:0;f30636bb2128:38071 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:02,196 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T11:48:02,196 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73003573, jitterRate=0.08783800899982452}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-07T11:48:02,197 INFO [RS:1;f30636bb2128:40315 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-07T11:48:02,198 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1730980082123Initializing all the Stores at 1730980082124 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730980082124Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730980082124Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730980082124Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730980082124Cleaning up temporary data from old regions at 1730980082181 (+57 ms)Region opened successfully at 1730980082198 (+17 ms) 2024-11-07T11:48:02,198 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-07T11:48:02,198 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-07T11:48:02,198 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-07T11:48:02,198 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-07T11:48:02,198 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-07T11:48:02,200 INFO [RS:0;f30636bb2128:38071 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-07T11:48:02,203 INFO [RS:0;f30636bb2128:38071 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-07T11:48:02,203 INFO [RS:1;f30636bb2128:40315 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-07T11:48:02,205 INFO [RS:0;f30636bb2128:38071 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:02,205 INFO [RS:1;f30636bb2128:40315 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:02,205 DEBUG [RS:1;f30636bb2128:40315 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:02,205 DEBUG [RS:0;f30636bb2128:38071 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:02,205 DEBUG [RS:1;f30636bb2128:40315 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:02,205 DEBUG [RS:0;f30636bb2128:38071 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:02,206 DEBUG [RS:1;f30636bb2128:40315 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:02,206 DEBUG [RS:0;f30636bb2128:38071 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:02,206 DEBUG [RS:1;f30636bb2128:40315 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:02,206 DEBUG [RS:0;f30636bb2128:38071 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:02,206 DEBUG [RS:1;f30636bb2128:40315 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:02,206 DEBUG [RS:0;f30636bb2128:38071 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:02,206 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-07T11:48:02,206 DEBUG [RS:0;f30636bb2128:38071 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f30636bb2128:0, corePoolSize=2, maxPoolSize=2 2024-11-07T11:48:02,206 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1730980082198Disabling compacts and flushes for region at 1730980082198Disabling writes for close at 1730980082198Writing region close event to WAL at 1730980082206 (+8 ms)Closed at 1730980082206 2024-11-07T11:48:02,206 DEBUG [RS:1;f30636bb2128:40315 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f30636bb2128:0, corePoolSize=2, maxPoolSize=2 2024-11-07T11:48:02,206 DEBUG [RS:0;f30636bb2128:38071 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:02,207 DEBUG [RS:1;f30636bb2128:40315 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:02,207 DEBUG [RS:0;f30636bb2128:38071 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:02,207 DEBUG [RS:1;f30636bb2128:40315 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:02,207 DEBUG [RS:0;f30636bb2128:38071 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:02,207 DEBUG [RS:1;f30636bb2128:40315 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:02,207 DEBUG [RS:0;f30636bb2128:38071 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:02,207 DEBUG [RS:1;f30636bb2128:40315 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:02,207 DEBUG [RS:0;f30636bb2128:38071 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:02,207 DEBUG [RS:1;f30636bb2128:40315 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:02,207 DEBUG [RS:0;f30636bb2128:38071 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:02,208 DEBUG [RS:1;f30636bb2128:40315 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:02,208 DEBUG [RS:0;f30636bb2128:38071 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f30636bb2128:0, corePoolSize=3, maxPoolSize=3 2024-11-07T11:48:02,208 DEBUG [RS:0;f30636bb2128:38071 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f30636bb2128:0, corePoolSize=3, maxPoolSize=3 2024-11-07T11:48:02,208 DEBUG [RS:1;f30636bb2128:40315 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f30636bb2128:0, corePoolSize=3, maxPoolSize=3 2024-11-07T11:48:02,208 DEBUG [RS:1;f30636bb2128:40315 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f30636bb2128:0, corePoolSize=3, maxPoolSize=3 2024-11-07T11:48:02,209 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-07T11:48:02,209 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-07T11:48:02,212 INFO [RS:1;f30636bb2128:40315 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:02,213 INFO [RS:0;f30636bb2128:38071 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:02,213 INFO [RS:1;f30636bb2128:40315 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:02,213 INFO [RS:0;f30636bb2128:38071 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:02,213 INFO [RS:1;f30636bb2128:40315 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:02,213 INFO [RS:0;f30636bb2128:38071 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:02,213 INFO [RS:1;f30636bb2128:40315 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:02,213 INFO [RS:0;f30636bb2128:38071 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:02,213 INFO [RS:1;f30636bb2128:40315 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:02,213 INFO [RS:0;f30636bb2128:38071 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:02,213 INFO [RS:1;f30636bb2128:40315 {}] hbase.ChoreService(168): Chore ScheduledChore name=f30636bb2128,40315,1730980081104-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-07T11:48:02,213 INFO [RS:0;f30636bb2128:38071 {}] hbase.ChoreService(168): Chore ScheduledChore name=f30636bb2128,38071,1730980080953-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-07T11:48:02,217 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-07T11:48:02,227 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T11:48:02,230 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-07T11:48:02,244 INFO [RS:1;f30636bb2128:40315 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-07T11:48:02,247 INFO [RS:1;f30636bb2128:40315 {}] hbase.ChoreService(168): Chore ScheduledChore name=f30636bb2128,40315,1730980081104-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:02,247 INFO [RS:0;f30636bb2128:38071 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-07T11:48:02,247 INFO [RS:0;f30636bb2128:38071 {}] hbase.ChoreService(168): Chore ScheduledChore name=f30636bb2128,38071,1730980080953-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:02,247 INFO [RS:1;f30636bb2128:40315 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:02,247 INFO [RS:1;f30636bb2128:40315 {}] regionserver.Replication(171): f30636bb2128,40315,1730980081104 started 2024-11-07T11:48:02,247 INFO [RS:0;f30636bb2128:38071 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:02,248 INFO [RS:0;f30636bb2128:38071 {}] regionserver.Replication(171): f30636bb2128,38071,1730980080953 started 2024-11-07T11:48:02,276 INFO [RS:0;f30636bb2128:38071 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:02,276 INFO [RS:1;f30636bb2128:40315 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:02,277 INFO [RS:0;f30636bb2128:38071 {}] regionserver.HRegionServer(1482): Serving as f30636bb2128,38071,1730980080953, RpcServer on f30636bb2128/172.17.0.2:38071, sessionid=0x1001a11f38e0001 2024-11-07T11:48:02,277 INFO [RS:1;f30636bb2128:40315 {}] regionserver.HRegionServer(1482): Serving as f30636bb2128,40315,1730980081104, RpcServer on f30636bb2128/172.17.0.2:40315, sessionid=0x1001a11f38e0002 2024-11-07T11:48:02,278 DEBUG [RS:0;f30636bb2128:38071 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-07T11:48:02,278 DEBUG [RS:1;f30636bb2128:40315 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-07T11:48:02,278 DEBUG [RS:1;f30636bb2128:40315 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f30636bb2128,40315,1730980081104 2024-11-07T11:48:02,278 DEBUG [RS:0;f30636bb2128:38071 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f30636bb2128,38071,1730980080953 2024-11-07T11:48:02,279 DEBUG [RS:1;f30636bb2128:40315 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f30636bb2128,40315,1730980081104' 2024-11-07T11:48:02,279 DEBUG [RS:0;f30636bb2128:38071 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f30636bb2128,38071,1730980080953' 2024-11-07T11:48:02,279 DEBUG [RS:0;f30636bb2128:38071 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-07T11:48:02,279 DEBUG [RS:1;f30636bb2128:40315 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-07T11:48:02,280 DEBUG [RS:1;f30636bb2128:40315 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-07T11:48:02,280 DEBUG [RS:0;f30636bb2128:38071 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-07T11:48:02,281 DEBUG [RS:0;f30636bb2128:38071 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-07T11:48:02,281 DEBUG [RS:0;f30636bb2128:38071 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-07T11:48:02,281 DEBUG [RS:1;f30636bb2128:40315 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-07T11:48:02,281 DEBUG [RS:1;f30636bb2128:40315 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-07T11:48:02,281 DEBUG [RS:1;f30636bb2128:40315 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f30636bb2128,40315,1730980081104 2024-11-07T11:48:02,281 DEBUG [RS:0;f30636bb2128:38071 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f30636bb2128,38071,1730980080953 2024-11-07T11:48:02,281 DEBUG [RS:0;f30636bb2128:38071 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f30636bb2128,38071,1730980080953' 2024-11-07T11:48:02,281 DEBUG [RS:1;f30636bb2128:40315 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f30636bb2128,40315,1730980081104' 2024-11-07T11:48:02,281 DEBUG [RS:0;f30636bb2128:38071 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-07T11:48:02,281 DEBUG [RS:1;f30636bb2128:40315 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-07T11:48:02,282 DEBUG [RS:1;f30636bb2128:40315 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-07T11:48:02,282 DEBUG [RS:0;f30636bb2128:38071 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-07T11:48:02,283 DEBUG [RS:1;f30636bb2128:40315 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-07T11:48:02,283 INFO [RS:1;f30636bb2128:40315 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-07T11:48:02,283 INFO [RS:1;f30636bb2128:40315 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-07T11:48:02,283 DEBUG [RS:0;f30636bb2128:38071 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-07T11:48:02,283 INFO [RS:0;f30636bb2128:38071 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-07T11:48:02,283 INFO [RS:0;f30636bb2128:38071 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-07T11:48:02,381 WARN [f30636bb2128:35319 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-07T11:48:02,389 INFO [RS:0;f30636bb2128:38071 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-07T11:48:02,389 INFO [RS:1;f30636bb2128:40315 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-07T11:48:02,390 INFO [RS:0;f30636bb2128:38071 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=f30636bb2128%2C38071%2C1730980080953, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/WALs/f30636bb2128,38071,1730980080953, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/oldWALs, maxLogs=55 2024-11-07T11:48:02,391 INFO [RS:1;f30636bb2128:40315 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=f30636bb2128%2C40315%2C1730980081104, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/WALs/f30636bb2128,40315,1730980081104, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/oldWALs, maxLogs=55 2024-11-07T11:48:02,418 INFO [RS:1;f30636bb2128:40315 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/WALs/f30636bb2128,40315,1730980081104/f30636bb2128%2C40315%2C1730980081104.1730980082395 2024-11-07T11:48:02,419 DEBUG [RS:1;f30636bb2128:40315 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-07T11:48:02,423 INFO [RS:0;f30636bb2128:38071 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/WALs/f30636bb2128,38071,1730980080953/f30636bb2128%2C38071%2C1730980080953.1730980082395 2024-11-07T11:48:02,423 DEBUG [RS:0;f30636bb2128:38071 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-07T11:48:02,634 DEBUG [f30636bb2128:35319 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-07T11:48:02,646 DEBUG [f30636bb2128:35319 {}] balancer.BalancerClusterState(204): Hosts are {f30636bb2128=0} racks are {/default-rack=0} 2024-11-07T11:48:02,651 DEBUG [f30636bb2128:35319 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-07T11:48:02,651 DEBUG [f30636bb2128:35319 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-07T11:48:02,651 DEBUG [f30636bb2128:35319 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-07T11:48:02,651 DEBUG [f30636bb2128:35319 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-07T11:48:02,651 INFO [f30636bb2128:35319 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-07T11:48:02,651 INFO [f30636bb2128:35319 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-07T11:48:02,651 DEBUG [f30636bb2128:35319 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-07T11:48:02,659 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=f30636bb2128,38071,1730980080953 2024-11-07T11:48:02,665 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f30636bb2128,38071,1730980080953, state=OPENING 2024-11-07T11:48:02,670 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-07T11:48:02,672 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35319-0x1001a11f38e0000, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:48:02,672 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40315-0x1001a11f38e0002, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:48:02,672 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38071-0x1001a11f38e0001, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:48:02,673 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:48:02,673 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:48:02,673 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:48:02,674 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T11:48:02,676 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=f30636bb2128,38071,1730980080953}] 2024-11-07T11:48:02,849 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-07T11:48:02,852 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33013, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-07T11:48:02,861 INFO [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-07T11:48:02,861 INFO [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T11:48:02,862 INFO [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-07T11:48:02,863 INFO [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=f30636bb2128%2C38071%2C1730980080953.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/WALs/f30636bb2128,38071,1730980080953, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/oldWALs, maxLogs=55 2024-11-07T11:48:02,883 INFO [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/WALs/f30636bb2128,38071,1730980080953/f30636bb2128%2C38071%2C1730980080953.meta.1730980082865.meta 2024-11-07T11:48:02,884 DEBUG [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-07T11:48:02,884 DEBUG [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-07T11:48:02,886 DEBUG [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-07T11:48:02,888 DEBUG [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-07T11:48:02,893 INFO [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-07T11:48:02,897 DEBUG [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-07T11:48:02,897 DEBUG [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:48:02,898 DEBUG [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-07T11:48:02,898 DEBUG [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-07T11:48:02,899 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-07T11:48:02,901 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-07T11:48:02,901 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:48:02,901 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T11:48:02,902 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-07T11:48:02,903 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-07T11:48:02,903 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:48:02,903 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T11:48:02,904 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-07T11:48:02,905 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-07T11:48:02,905 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:48:02,906 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T11:48:02,906 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-07T11:48:02,907 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-07T11:48:02,907 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:48:02,907 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T11:48:02,908 DEBUG [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-07T11:48:02,909 DEBUG [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740 2024-11-07T11:48:02,910 DEBUG [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740 2024-11-07T11:48:02,910 DEBUG [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-07T11:48:02,910 DEBUG [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-07T11:48:02,911 DEBUG [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-07T11:48:02,912 DEBUG [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-07T11:48:02,913 INFO [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71671224, jitterRate=0.06798446178436279}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-07T11:48:02,913 DEBUG [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-07T11:48:02,915 DEBUG [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1730980082898Writing region info on filesystem at 1730980082899 (+1 ms)Initializing all the Stores at 1730980082899Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730980082899Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730980082899Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730980082899Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730980082899Cleaning up temporary data from old regions at 1730980082911 (+12 ms)Running coprocessor post-open hooks at 1730980082913 (+2 ms)Region opened successfully at 1730980082915 (+2 ms) 2024-11-07T11:48:02,923 INFO [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1730980082840 2024-11-07T11:48:02,935 DEBUG [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-07T11:48:02,936 INFO [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-07T11:48:02,937 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=f30636bb2128,38071,1730980080953 2024-11-07T11:48:02,938 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f30636bb2128,38071,1730980080953, state=OPEN 2024-11-07T11:48:02,941 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40315-0x1001a11f38e0002, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:48:02,941 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38071-0x1001a11f38e0001, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:48:02,941 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35319-0x1001a11f38e0000, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:48:02,941 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:48:02,941 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:48:02,941 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:48:02,941 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=f30636bb2128,38071,1730980080953 2024-11-07T11:48:02,944 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-07T11:48:02,944 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f30636bb2128,38071,1730980080953 in 265 msec 2024-11-07T11:48:02,950 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-07T11:48:02,950 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 729 msec 2024-11-07T11:48:02,951 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-07T11:48:02,951 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-07T11:48:02,976 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T11:48:02,978 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f30636bb2128,38071,1730980080953, seqNum=-1] 2024-11-07T11:48:02,998 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T11:48:03,001 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56617, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T11:48:03,019 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0680 sec 2024-11-07T11:48:03,020 INFO [master/f30636bb2128:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1730980083020, completionTime=-1 2024-11-07T11:48:03,022 INFO [master/f30636bb2128:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-07T11:48:03,023 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-07T11:48:03,049 INFO [master/f30636bb2128:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-07T11:48:03,049 INFO [master/f30636bb2128:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1730980143049 2024-11-07T11:48:03,050 INFO [master/f30636bb2128:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1730980203050 2024-11-07T11:48:03,050 INFO [master/f30636bb2128:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 27 msec 2024-11-07T11:48:03,053 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-07T11:48:03,061 INFO [master/f30636bb2128:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f30636bb2128,35319,1730980080176-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:03,062 INFO [master/f30636bb2128:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f30636bb2128,35319,1730980080176-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:03,062 INFO [master/f30636bb2128:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f30636bb2128,35319,1730980080176-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:03,064 INFO [master/f30636bb2128:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-f30636bb2128:35319, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:03,064 INFO [master/f30636bb2128:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:03,065 INFO [master/f30636bb2128:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:03,070 DEBUG [master/f30636bb2128:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-07T11:48:03,096 INFO [master/f30636bb2128:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.894sec 2024-11-07T11:48:03,098 INFO [master/f30636bb2128:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-07T11:48:03,100 INFO [master/f30636bb2128:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-07T11:48:03,101 INFO [master/f30636bb2128:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-07T11:48:03,101 INFO [master/f30636bb2128:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-07T11:48:03,101 INFO [master/f30636bb2128:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-07T11:48:03,102 INFO [master/f30636bb2128:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f30636bb2128,35319,1730980080176-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-07T11:48:03,102 INFO [master/f30636bb2128:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f30636bb2128,35319,1730980080176-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-07T11:48:03,104 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-07T11:48:03,104 INFO [master/f30636bb2128:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-07T11:48:03,105 INFO [master/f30636bb2128:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f30636bb2128,35319,1730980080176-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:03,156 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71985066, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T11:48:03,158 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-07T11:48:03,158 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-07T11:48:03,162 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f30636bb2128,35319,-1 for getting cluster id 2024-11-07T11:48:03,165 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T11:48:03,194 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '69e30f5c-659f-49ac-a3c3-809bd52e6096' 2024-11-07T11:48:03,200 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T11:48:03,200 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "69e30f5c-659f-49ac-a3c3-809bd52e6096" 2024-11-07T11:48:03,202 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ea769ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T11:48:03,203 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f30636bb2128,35319,-1] 2024-11-07T11:48:03,206 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T11:48:03,207 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T11:48:03,209 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37372, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T11:48:03,211 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e124b8e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T11:48:03,212 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T11:48:03,218 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f30636bb2128,38071,1730980080953, seqNum=-1] 2024-11-07T11:48:03,219 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T11:48:03,222 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50698, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T11:48:03,247 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=f30636bb2128,35319,1730980080176 2024-11-07T11:48:03,252 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T11:48:03,256 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is f30636bb2128,35319,1730980080176 2024-11-07T11:48:03,259 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1b99b208 2024-11-07T11:48:03,260 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T11:48:03,263 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37384, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T11:48:03,269 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35319 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1730980083247', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T11:48:03,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35319 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1730980083247 2024-11-07T11:48:03,280 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1730980083247 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T11:48:03,283 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35319 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1730980083247" procId is: 4 2024-11-07T11:48:03,283 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:48:03,284 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1730980083247 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T11:48:03,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T11:48:03,300 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1730980083247-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 85e3a3676af55f05d5f68bda5ac09869, NAME => 'testBlockEvictionOnRegionMove1730980083247,,1730980083264.85e3a3676af55f05d5f68bda5ac09869.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1730980083247', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47 2024-11-07T11:48:03,307 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1730980083247-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1730980083247,,1730980083264.85e3a3676af55f05d5f68bda5ac09869.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:48:03,307 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1730980083247-pool-0 {}] regionserver.HRegion(1722): Closing 85e3a3676af55f05d5f68bda5ac09869, disabling compactions & flushes 2024-11-07T11:48:03,307 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1730980083247-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1730980083247,,1730980083264.85e3a3676af55f05d5f68bda5ac09869. 2024-11-07T11:48:03,307 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1730980083247-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1730980083247,,1730980083264.85e3a3676af55f05d5f68bda5ac09869. 2024-11-07T11:48:03,307 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1730980083247-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1730980083247,,1730980083264.85e3a3676af55f05d5f68bda5ac09869. after waiting 0 ms 2024-11-07T11:48:03,307 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1730980083247-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1730980083247,,1730980083264.85e3a3676af55f05d5f68bda5ac09869. 2024-11-07T11:48:03,307 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1730980083247-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1730980083247,,1730980083264.85e3a3676af55f05d5f68bda5ac09869. 2024-11-07T11:48:03,307 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1730980083247-pool-0 {}] regionserver.HRegion(1676): Region close journal for 85e3a3676af55f05d5f68bda5ac09869: Waiting for close lock at 1730980083307Disabling compacts and flushes for region at 1730980083307Disabling writes for close at 1730980083307Writing region close event to WAL at 1730980083307Closed at 1730980083307 2024-11-07T11:48:03,309 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1730980083247 execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T11:48:03,313 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1730980083247,,1730980083264.85e3a3676af55f05d5f68bda5ac09869.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1730980083309"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730980083309"}]},"ts":"1730980083309"} 2024-11-07T11:48:03,318 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-07T11:48:03,319 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1730980083247 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T11:48:03,321 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1730980083247","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730980083319"}]},"ts":"1730980083319"} 2024-11-07T11:48:03,325 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1730980083247, state=ENABLING in hbase:meta 2024-11-07T11:48:03,326 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {f30636bb2128=0} racks are {/default-rack=0} 2024-11-07T11:48:03,326 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-07T11:48:03,326 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-07T11:48:03,326 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-07T11:48:03,326 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-07T11:48:03,326 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-07T11:48:03,326 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-07T11:48:03,326 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-07T11:48:03,328 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1730980083247, region=85e3a3676af55f05d5f68bda5ac09869, ASSIGN}] 2024-11-07T11:48:03,329 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1730980083247, region=85e3a3676af55f05d5f68bda5ac09869, ASSIGN 2024-11-07T11:48:03,330 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1730980083247, region=85e3a3676af55f05d5f68bda5ac09869, ASSIGN; state=OFFLINE, location=f30636bb2128,40315,1730980081104; forceNewPlan=false, retain=false 2024-11-07T11:48:03,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T11:48:03,483 INFO [f30636bb2128:35319 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-07T11:48:03,483 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=85e3a3676af55f05d5f68bda5ac09869, regionState=OPENING, regionLocation=f30636bb2128,40315,1730980081104 2024-11-07T11:48:03,487 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1730980083247, region=85e3a3676af55f05d5f68bda5ac09869, ASSIGN because future has completed 2024-11-07T11:48:03,489 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 85e3a3676af55f05d5f68bda5ac09869, server=f30636bb2128,40315,1730980081104}] 2024-11-07T11:48:03,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T11:48:03,642 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-07T11:48:03,644 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44533, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-07T11:48:03,647 INFO [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1730980083247,,1730980083264.85e3a3676af55f05d5f68bda5ac09869. 2024-11-07T11:48:03,648 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 85e3a3676af55f05d5f68bda5ac09869, NAME => 'testBlockEvictionOnRegionMove1730980083247,,1730980083264.85e3a3676af55f05d5f68bda5ac09869.', STARTKEY => '', ENDKEY => ''} 2024-11-07T11:48:03,648 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1730980083247 85e3a3676af55f05d5f68bda5ac09869 2024-11-07T11:48:03,649 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1730980083247,,1730980083264.85e3a3676af55f05d5f68bda5ac09869.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:48:03,649 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 85e3a3676af55f05d5f68bda5ac09869 2024-11-07T11:48:03,649 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 85e3a3676af55f05d5f68bda5ac09869 2024-11-07T11:48:03,650 INFO [StoreOpener-85e3a3676af55f05d5f68bda5ac09869-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 85e3a3676af55f05d5f68bda5ac09869 2024-11-07T11:48:03,654 INFO [StoreOpener-85e3a3676af55f05d5f68bda5ac09869-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 85e3a3676af55f05d5f68bda5ac09869 columnFamilyName family 2024-11-07T11:48:03,654 DEBUG [StoreOpener-85e3a3676af55f05d5f68bda5ac09869-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:48:03,654 INFO [StoreOpener-85e3a3676af55f05d5f68bda5ac09869-1 {}] regionserver.HStore(327): Store=85e3a3676af55f05d5f68bda5ac09869/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T11:48:03,655 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 85e3a3676af55f05d5f68bda5ac09869 2024-11-07T11:48:03,656 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/default/testBlockEvictionOnRegionMove1730980083247/85e3a3676af55f05d5f68bda5ac09869 2024-11-07T11:48:03,656 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/default/testBlockEvictionOnRegionMove1730980083247/85e3a3676af55f05d5f68bda5ac09869 2024-11-07T11:48:03,656 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 85e3a3676af55f05d5f68bda5ac09869 2024-11-07T11:48:03,657 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 85e3a3676af55f05d5f68bda5ac09869 2024-11-07T11:48:03,657 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 85e3a3676af55f05d5f68bda5ac09869 2024-11-07T11:48:03,665 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/default/testBlockEvictionOnRegionMove1730980083247/85e3a3676af55f05d5f68bda5ac09869/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T11:48:03,665 INFO [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 85e3a3676af55f05d5f68bda5ac09869; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65856045, jitterRate=-0.018668457865715027}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T11:48:03,666 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 85e3a3676af55f05d5f68bda5ac09869 2024-11-07T11:48:03,667 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 85e3a3676af55f05d5f68bda5ac09869: Running coprocessor pre-open hook at 1730980083649Writing region info on filesystem at 1730980083649Initializing all the Stores at 1730980083649Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730980083649Cleaning up temporary data from old regions at 1730980083657 (+8 ms)Running coprocessor post-open hooks at 1730980083666 (+9 ms)Region opened successfully at 1730980083667 (+1 ms) 2024-11-07T11:48:03,668 INFO [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1730980083247,,1730980083264.85e3a3676af55f05d5f68bda5ac09869., pid=6, masterSystemTime=1730980083642 2024-11-07T11:48:03,671 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1730980083247,,1730980083264.85e3a3676af55f05d5f68bda5ac09869. 2024-11-07T11:48:03,672 INFO [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1730980083247,,1730980083264.85e3a3676af55f05d5f68bda5ac09869. 2024-11-07T11:48:03,672 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=85e3a3676af55f05d5f68bda5ac09869, regionState=OPEN, openSeqNum=2, regionLocation=f30636bb2128,40315,1730980081104 2024-11-07T11:48:03,675 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35319 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=f30636bb2128,40315,1730980081104, table=testBlockEvictionOnRegionMove1730980083247, region=85e3a3676af55f05d5f68bda5ac09869. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-07T11:48:03,676 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 85e3a3676af55f05d5f68bda5ac09869, server=f30636bb2128,40315,1730980081104 because future has completed 2024-11-07T11:48:03,680 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-07T11:48:03,680 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 85e3a3676af55f05d5f68bda5ac09869, server=f30636bb2128,40315,1730980081104 in 188 msec 2024-11-07T11:48:03,682 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-07T11:48:03,682 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1730980083247, region=85e3a3676af55f05d5f68bda5ac09869, ASSIGN in 353 msec 2024-11-07T11:48:03,683 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1730980083247 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T11:48:03,683 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1730980083247","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730980083683"}]},"ts":"1730980083683"} 2024-11-07T11:48:03,686 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1730980083247, state=ENABLED in hbase:meta 2024-11-07T11:48:03,687 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1730980083247 execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T11:48:03,689 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1730980083247 in 415 msec 2024-11-07T11:48:03,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35319 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T11:48:03,916 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1730980083247 get assigned. Timeout = 60000ms 2024-11-07T11:48:03,916 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1730980083247 completed 2024-11-07T11:48:03,919 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T11:48:03,925 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1730980083247 assigned to meta. Checking AM states. 2024-11-07T11:48:03,925 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T11:48:03,926 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1730980083247 assigned. 2024-11-07T11:48:03,933 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1730980083247', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1730980083247,,1730980083264.85e3a3676af55f05d5f68bda5ac09869., hostname=f30636bb2128,40315,1730980081104, seqNum=2] 2024-11-07T11:48:03,934 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T11:48:03,937 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56470, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T11:48:03,946 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 85e3a3676af55f05d5f68bda5ac09869 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-07T11:48:03,970 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 935687fdb97a471d9b6d0f2b6baf342d 2024-11-07T11:48:04,009 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/default/testBlockEvictionOnRegionMove1730980083247/85e3a3676af55f05d5f68bda5ac09869/.tmp/family/935687fdb97a471d9b6d0f2b6baf342d is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-07T11:48:04,014 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/default/testBlockEvictionOnRegionMove1730980083247/85e3a3676af55f05d5f68bda5ac09869/.tmp/family/935687fdb97a471d9b6d0f2b6baf342d 2024-11-07T11:48:04,050 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/default/testBlockEvictionOnRegionMove1730980083247/85e3a3676af55f05d5f68bda5ac09869/.tmp/family/935687fdb97a471d9b6d0f2b6baf342d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/default/testBlockEvictionOnRegionMove1730980083247/85e3a3676af55f05d5f68bda5ac09869/family/935687fdb97a471d9b6d0f2b6baf342d 2024-11-07T11:48:04,051 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/default/testBlockEvictionOnRegionMove1730980083247/85e3a3676af55f05d5f68bda5ac09869/family/935687fdb97a471d9b6d0f2b6baf342d, delay=986 ms 2024-11-07T11:48:04,053 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/default/testBlockEvictionOnRegionMove1730980083247/85e3a3676af55f05d5f68bda5ac09869/family/935687fdb97a471d9b6d0f2b6baf342d, entries=2, sequenceid=6, filesize=4.9 K 2024-11-07T11:48:04,060 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 85e3a3676af55f05d5f68bda5ac09869 in 112ms, sequenceid=6, compaction requested=false 2024-11-07T11:48:04,061 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1730980083247' 2024-11-07T11:48:04,063 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 85e3a3676af55f05d5f68bda5ac09869: 2024-11-07T11:48:05,040 DEBUG [hfile-prefetch-1730980084051 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 935687fdb97a471d9b6d0f2b6baf342d_0. 2024-11-07T11:48:05,045 DEBUG [hfile-prefetch-1730980084051 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 935687fdb97a471d9b6d0f2b6baf342d_117. 2024-11-07T11:48:05,046 DEBUG [hfile-prefetch-1730980084051 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/default/testBlockEvictionOnRegionMove1730980083247/85e3a3676af55f05d5f68bda5ac09869/family/935687fdb97a471d9b6d0f2b6baf342d, with total blocks 2, and data blocks 1 2024-11-07T11:48:05,046 DEBUG [hfile-prefetch-1730980084051 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-07T11:48:05,046 DEBUG [hfile-prefetch-1730980084051 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/default/testBlockEvictionOnRegionMove1730980083247/85e3a3676af55f05d5f68bda5ac09869/family/935687fdb97a471d9b6d0f2b6baf342d, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-07T11:48:05,046 DEBUG [hfile-prefetch-1730980084051 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/default/testBlockEvictionOnRegionMove1730980083247/85e3a3676af55f05d5f68bda5ac09869/family/935687fdb97a471d9b6d0f2b6baf342d. Will sleep 100ms and try the verification again. 2024-11-07T11:48:05,073 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35319 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=85e3a3676af55f05d5f68bda5ac09869, source=f30636bb2128,40315,1730980081104, destination=f30636bb2128,38071,1730980080953, warming up region on f30636bb2128,38071,1730980080953 2024-11-07T11:48:05,075 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35319 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=85e3a3676af55f05d5f68bda5ac09869, source=f30636bb2128,40315,1730980081104, destination=f30636bb2128,38071,1730980080953, running balancer 2024-11-07T11:48:05,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35319 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1730980083247, region=85e3a3676af55f05d5f68bda5ac09869, REOPEN/MOVE 2024-11-07T11:48:05,077 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1730980083247, region=85e3a3676af55f05d5f68bda5ac09869, REOPEN/MOVE 2024-11-07T11:48:05,077 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38071 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1730980083247,,1730980083264.85e3a3676af55f05d5f68bda5ac09869. 2024-11-07T11:48:05,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38071 {}] regionserver.HRegion(7855): Warmup {ENCODED => 85e3a3676af55f05d5f68bda5ac09869, NAME => 'testBlockEvictionOnRegionMove1730980083247,,1730980083264.85e3a3676af55f05d5f68bda5ac09869.', STARTKEY => '', ENDKEY => ''} 2024-11-07T11:48:05,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38071 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1730980083247,,1730980083264.85e3a3676af55f05d5f68bda5ac09869.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:48:05,079 INFO [StoreOpener-85e3a3676af55f05d5f68bda5ac09869-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 85e3a3676af55f05d5f68bda5ac09869 2024-11-07T11:48:05,079 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=85e3a3676af55f05d5f68bda5ac09869, regionState=CLOSING, regionLocation=f30636bb2128,40315,1730980081104 2024-11-07T11:48:05,079 INFO [StoreOpener-85e3a3676af55f05d5f68bda5ac09869-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 85e3a3676af55f05d5f68bda5ac09869 columnFamilyName family 2024-11-07T11:48:05,079 DEBUG [StoreOpener-85e3a3676af55f05d5f68bda5ac09869-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:48:05,083 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1730980083247, region=85e3a3676af55f05d5f68bda5ac09869, REOPEN/MOVE because future has completed 2024-11-07T11:48:05,083 DEBUG [StoreOpener-85e3a3676af55f05d5f68bda5ac09869-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/default/testBlockEvictionOnRegionMove1730980083247/85e3a3676af55f05d5f68bda5ac09869/family/935687fdb97a471d9b6d0f2b6baf342d 2024-11-07T11:48:05,083 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-07T11:48:05,083 INFO [StoreOpener-85e3a3676af55f05d5f68bda5ac09869-1 {}] regionserver.HStore(327): Store=85e3a3676af55f05d5f68bda5ac09869/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T11:48:05,083 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 85e3a3676af55f05d5f68bda5ac09869, server=f30636bb2128,40315,1730980081104}] 2024-11-07T11:48:05,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38071 {}] regionserver.HRegion(1722): Closing 85e3a3676af55f05d5f68bda5ac09869, disabling compactions & flushes 2024-11-07T11:48:05,084 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38071 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1730980083247,,1730980083264.85e3a3676af55f05d5f68bda5ac09869. 2024-11-07T11:48:05,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38071 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1730980083247,,1730980083264.85e3a3676af55f05d5f68bda5ac09869. 2024-11-07T11:48:05,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38071 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1730980083247,,1730980083264.85e3a3676af55f05d5f68bda5ac09869. after waiting 0 ms 2024-11-07T11:48:05,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38071 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1730980083247,,1730980083264.85e3a3676af55f05d5f68bda5ac09869. 2024-11-07T11:48:05,085 DEBUG [StoreFileCloser-85e3a3676af55f05d5f68bda5ac09869-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/default/testBlockEvictionOnRegionMove1730980083247/85e3a3676af55f05d5f68bda5ac09869/family/935687fdb97a471d9b6d0f2b6baf342d 2024-11-07T11:48:05,086 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38071 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1730980083247,,1730980083264.85e3a3676af55f05d5f68bda5ac09869. 2024-11-07T11:48:05,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38071 {}] regionserver.HRegion(1676): Region close journal for 85e3a3676af55f05d5f68bda5ac09869: Waiting for close lock at 1730980085084Disabling compacts and flushes for region at 1730980085084Disabling writes for close at 1730980085084Writing region close event to WAL at 1730980085086 (+2 ms)Closed at 1730980085086 2024-11-07T11:48:05,087 WARN [hfile-prefetch-1730980084051 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/default/testBlockEvictionOnRegionMove1730980083247/85e3a3676af55f05d5f68bda5ac09869/family/935687fdb97a471d9b6d0f2b6baf342d, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2338) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2321) ~[classes/:?] ... 9 more 2024-11-07T11:48:05,088 DEBUG [hfile-prefetch-1730980084051 {}] hfile.PrefetchExecutor(121): Prefetch completed for 935687fdb97a471d9b6d0f2b6baf342d 2024-11-07T11:48:05,126 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-07T11:48:05,127 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-07T11:48:05,127 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-07T11:48:05,127 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-07T11:48:05,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35319 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-07T11:48:05,144 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-07T11:48:05,144 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-07T11:48:05,144 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-07T11:48:05,144 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-07T11:48:05,147 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-07T11:48:05,147 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1730980085127 2024-11-07T11:48:05,147 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-07T11:48:05,147 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-07T11:48:05,147 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1730980085144 2024-11-07T11:48:05,147 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-07T11:48:05,242 INFO [RS_CLOSE_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 85e3a3676af55f05d5f68bda5ac09869 2024-11-07T11:48:05,242 DEBUG [RS_CLOSE_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-07T11:48:05,243 DEBUG [RS_CLOSE_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 85e3a3676af55f05d5f68bda5ac09869, disabling compactions & flushes 2024-11-07T11:48:05,243 INFO [RS_CLOSE_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1730980083247,,1730980083264.85e3a3676af55f05d5f68bda5ac09869. 2024-11-07T11:48:05,243 DEBUG [RS_CLOSE_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1730980083247,,1730980083264.85e3a3676af55f05d5f68bda5ac09869. 2024-11-07T11:48:05,243 DEBUG [RS_CLOSE_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1730980083247,,1730980083264.85e3a3676af55f05d5f68bda5ac09869. after waiting 0 ms 2024-11-07T11:48:05,243 DEBUG [RS_CLOSE_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1730980083247,,1730980083264.85e3a3676af55f05d5f68bda5ac09869. 2024-11-07T11:48:05,245 DEBUG [StoreFileCloser-85e3a3676af55f05d5f68bda5ac09869-family-1 {}] bucket.BucketCache(1864): found 1 blocks for file 935687fdb97a471d9b6d0f2b6baf342d, starting offset: 0, end offset: 9223372036854775807 2024-11-07T11:48:05,246 DEBUG [StoreFileCloser-85e3a3676af55f05d5f68bda5ac09869-family-1 {}] bucket.BucketCache(857): removed key 935687fdb97a471d9b6d0f2b6baf342d_0 from back map with offset lock 2094080 in the evict process 2024-11-07T11:48:05,247 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-07T11:48:05,247 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-07T11:48:05,247 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-07T11:48:05,247 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-07T11:48:05,248 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-07T11:48:05,248 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-07T11:48:05,248 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-07T11:48:05,248 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-07T11:48:05,251 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-07T11:48:05,251 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1730980085247 2024-11-07T11:48:05,251 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-07T11:48:05,252 DEBUG [RS_CLOSE_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/default/testBlockEvictionOnRegionMove1730980083247/85e3a3676af55f05d5f68bda5ac09869/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-07T11:48:05,253 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-07T11:48:05,253 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1730980085248 2024-11-07T11:48:05,253 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-07T11:48:05,254 INFO [RS_CLOSE_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1730980083247,,1730980083264.85e3a3676af55f05d5f68bda5ac09869. 2024-11-07T11:48:05,254 DEBUG [RS_CLOSE_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 85e3a3676af55f05d5f68bda5ac09869: Waiting for close lock at 1730980085243Running coprocessor pre-close hooks at 1730980085243Disabling compacts and flushes for region at 1730980085243Disabling writes for close at 1730980085243Writing region close event to WAL at 1730980085247 (+4 ms)Running coprocessor post-close hooks at 1730980085252 (+5 ms)Closed at 1730980085254 (+2 ms) 2024-11-07T11:48:05,254 INFO [RS_CLOSE_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 85e3a3676af55f05d5f68bda5ac09869 move to f30636bb2128,38071,1730980080953 record at close sequenceid=6 2024-11-07T11:48:05,258 INFO [RS_CLOSE_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 85e3a3676af55f05d5f68bda5ac09869 2024-11-07T11:48:05,258 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=85e3a3676af55f05d5f68bda5ac09869, regionState=CLOSED 2024-11-07T11:48:05,261 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 85e3a3676af55f05d5f68bda5ac09869, server=f30636bb2128,40315,1730980081104 because future has completed 2024-11-07T11:48:05,265 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-07T11:48:05,265 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 85e3a3676af55f05d5f68bda5ac09869, server=f30636bb2128,40315,1730980081104 in 180 msec 2024-11-07T11:48:05,266 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1730980083247, region=85e3a3676af55f05d5f68bda5ac09869, REOPEN/MOVE; state=CLOSED, location=f30636bb2128,38071,1730980080953; forceNewPlan=false, retain=false 2024-11-07T11:48:05,417 INFO [f30636bb2128:35319 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-07T11:48:05,417 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=85e3a3676af55f05d5f68bda5ac09869, regionState=OPENING, regionLocation=f30636bb2128,38071,1730980080953 2024-11-07T11:48:05,421 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1730980083247, region=85e3a3676af55f05d5f68bda5ac09869, REOPEN/MOVE because future has completed 2024-11-07T11:48:05,421 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 85e3a3676af55f05d5f68bda5ac09869, server=f30636bb2128,38071,1730980080953}] 2024-11-07T11:48:05,576 INFO [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1730980083247,,1730980083264.85e3a3676af55f05d5f68bda5ac09869. 2024-11-07T11:48:05,577 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 85e3a3676af55f05d5f68bda5ac09869, NAME => 'testBlockEvictionOnRegionMove1730980083247,,1730980083264.85e3a3676af55f05d5f68bda5ac09869.', STARTKEY => '', ENDKEY => ''} 2024-11-07T11:48:05,577 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1730980083247 85e3a3676af55f05d5f68bda5ac09869 2024-11-07T11:48:05,577 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1730980083247,,1730980083264.85e3a3676af55f05d5f68bda5ac09869.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:48:05,577 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 85e3a3676af55f05d5f68bda5ac09869 2024-11-07T11:48:05,577 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 85e3a3676af55f05d5f68bda5ac09869 2024-11-07T11:48:05,578 INFO [StoreOpener-85e3a3676af55f05d5f68bda5ac09869-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 85e3a3676af55f05d5f68bda5ac09869 2024-11-07T11:48:05,579 INFO [StoreOpener-85e3a3676af55f05d5f68bda5ac09869-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 85e3a3676af55f05d5f68bda5ac09869 columnFamilyName family 2024-11-07T11:48:05,579 DEBUG [StoreOpener-85e3a3676af55f05d5f68bda5ac09869-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:48:05,581 DEBUG [StoreFileOpener-85e3a3676af55f05d5f68bda5ac09869-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/default/testBlockEvictionOnRegionMove1730980083247/85e3a3676af55f05d5f68bda5ac09869/family/935687fdb97a471d9b6d0f2b6baf342d, delay=966 ms 2024-11-07T11:48:05,583 DEBUG [StoreOpener-85e3a3676af55f05d5f68bda5ac09869-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/default/testBlockEvictionOnRegionMove1730980083247/85e3a3676af55f05d5f68bda5ac09869/family/935687fdb97a471d9b6d0f2b6baf342d 2024-11-07T11:48:05,583 INFO [StoreOpener-85e3a3676af55f05d5f68bda5ac09869-1 {}] regionserver.HStore(327): Store=85e3a3676af55f05d5f68bda5ac09869/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T11:48:05,584 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 85e3a3676af55f05d5f68bda5ac09869 2024-11-07T11:48:05,584 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/default/testBlockEvictionOnRegionMove1730980083247/85e3a3676af55f05d5f68bda5ac09869 2024-11-07T11:48:05,585 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/default/testBlockEvictionOnRegionMove1730980083247/85e3a3676af55f05d5f68bda5ac09869 2024-11-07T11:48:05,585 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 85e3a3676af55f05d5f68bda5ac09869 2024-11-07T11:48:05,585 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 85e3a3676af55f05d5f68bda5ac09869 2024-11-07T11:48:05,590 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 85e3a3676af55f05d5f68bda5ac09869 2024-11-07T11:48:05,590 INFO [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 85e3a3676af55f05d5f68bda5ac09869; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70236248, jitterRate=0.04660165309906006}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T11:48:05,591 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 85e3a3676af55f05d5f68bda5ac09869 2024-11-07T11:48:05,591 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 85e3a3676af55f05d5f68bda5ac09869: Running coprocessor pre-open hook at 1730980085577Writing region info on filesystem at 1730980085577Initializing all the Stores at 1730980085578 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730980085578Cleaning up temporary data from old regions at 1730980085585 (+7 ms)Running coprocessor post-open hooks at 1730980085591 (+6 ms)Region opened successfully at 1730980085591 2024-11-07T11:48:05,592 INFO [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1730980083247,,1730980083264.85e3a3676af55f05d5f68bda5ac09869., pid=9, masterSystemTime=1730980085573 2024-11-07T11:48:05,595 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1730980083247,,1730980083264.85e3a3676af55f05d5f68bda5ac09869. 2024-11-07T11:48:05,596 INFO [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1730980083247,,1730980083264.85e3a3676af55f05d5f68bda5ac09869. 2024-11-07T11:48:05,596 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=85e3a3676af55f05d5f68bda5ac09869, regionState=OPEN, openSeqNum=10, regionLocation=f30636bb2128,38071,1730980080953 2024-11-07T11:48:05,599 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 85e3a3676af55f05d5f68bda5ac09869, server=f30636bb2128,38071,1730980080953 because future has completed 2024-11-07T11:48:05,603 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-07T11:48:05,603 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 85e3a3676af55f05d5f68bda5ac09869, server=f30636bb2128,38071,1730980080953 in 179 msec 2024-11-07T11:48:05,605 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1730980083247, region=85e3a3676af55f05d5f68bda5ac09869, REOPEN/MOVE in 528 msec 2024-11-07T11:48:05,629 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-07T11:48:05,630 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T11:48:05,630 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T11:48:05,635 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T11:48:05,636 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T11:48:05,636 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T11:48:05,636 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-07T11:48:05,636 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1596482003, stopped=false 2024-11-07T11:48:05,636 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=f30636bb2128,35319,1730980080176 2024-11-07T11:48:05,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40315-0x1001a11f38e0002, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-07T11:48:05,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35319-0x1001a11f38e0000, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-07T11:48:05,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38071-0x1001a11f38e0001, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-07T11:48:05,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40315-0x1001a11f38e0002, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:48:05,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35319-0x1001a11f38e0000, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:48:05,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38071-0x1001a11f38e0001, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:48:05,639 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T11:48:05,639 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T11:48:05,640 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T11:48:05,640 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T11:48:05,640 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35319-0x1001a11f38e0000, quorum=127.0.0.1:64253, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T11:48:05,640 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40315-0x1001a11f38e0002, quorum=127.0.0.1:64253, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T11:48:05,641 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38071-0x1001a11f38e0001, quorum=127.0.0.1:64253, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T11:48:05,641 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f30636bb2128,38071,1730980080953' ***** 2024-11-07T11:48:05,641 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-07T11:48:05,641 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f30636bb2128,40315,1730980081104' ***** 2024-11-07T11:48:05,641 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-07T11:48:05,641 INFO [RS:0;f30636bb2128:38071 {}] bucket.BucketCache(1739): Disabling cache 2024-11-07T11:48:05,641 INFO [RS:1;f30636bb2128:40315 {}] bucket.BucketCache(1739): Disabling cache 2024-11-07T11:48:05,642 INFO [RS:1;f30636bb2128:40315 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/bucket.persistence 2024-11-07T11:48:05,642 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-07T11:48:05,642 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-07T11:48:05,642 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-07T11:48:05,642 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-07T11:48:05,642 INFO [RS:0;f30636bb2128:38071 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/bucket.persistence 2024-11-07T11:48:05,642 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-07T11:48:05,642 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-07T11:48:05,642 INFO [RS:0;f30636bb2128:38071 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-07T11:48:05,642 INFO [RS:1;f30636bb2128:40315 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-07T11:48:05,642 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-07T11:48:05,642 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-07T11:48:05,652 DEBUG [RS:0;f30636bb2128:38071 {}] bucket.BucketCache(1488): Thread RS:0;f30636bb2128:38071 started persisting bucket cache to file 2024-11-07T11:48:05,653 DEBUG [RS:1;f30636bb2128:40315 {}] bucket.BucketCache(1488): Thread RS:1;f30636bb2128:40315 started persisting bucket cache to file 2024-11-07T11:48:05,653 DEBUG [RS:0;f30636bb2128:38071 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-07T11:48:05,653 DEBUG [RS:1;f30636bb2128:40315 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-07T11:48:05,653 DEBUG [RS:0;f30636bb2128:38071 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-07T11:48:05,653 DEBUG [RS:1;f30636bb2128:40315 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-07T11:48:05,656 DEBUG [RS:0;f30636bb2128:38071 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-07T11:48:05,656 DEBUG [RS:0;f30636bb2128:38071 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1730980085653 2024-11-07T11:48:05,656 DEBUG [RS:1;f30636bb2128:40315 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-07T11:48:05,656 DEBUG [RS:0;f30636bb2128:38071 {}] bucket.BucketCache(1510): Thread RS:0;f30636bb2128:38071 finished persisting bucket cache to file, renaming 2024-11-07T11:48:05,656 DEBUG [RS:1;f30636bb2128:40315 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1730980085653 2024-11-07T11:48:05,656 DEBUG [RS:1;f30636bb2128:40315 {}] bucket.BucketCache(1510): Thread RS:1;f30636bb2128:40315 finished persisting bucket cache to file, renaming 2024-11-07T11:48:05,656 WARN [RS:1;f30636bb2128:40315 {}] bucket.BucketCache(1513): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-07T11:48:05,656 INFO [RS:1;f30636bb2128:40315 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-07T11:48:05,657 INFO [RS:0;f30636bb2128:38071 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-07T11:48:05,657 INFO [RS:1;f30636bb2128:40315 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-07T11:48:05,657 INFO [RS:0;f30636bb2128:38071 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-07T11:48:05,657 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-07T11:48:05,657 INFO [RS:1;f30636bb2128:40315 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-07T11:48:05,657 INFO [RS:0;f30636bb2128:38071 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-07T11:48:05,657 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-07T11:48:05,657 INFO [RS:1;f30636bb2128:40315 {}] regionserver.HRegionServer(959): stopping server f30636bb2128,40315,1730980081104 2024-11-07T11:48:05,657 INFO [RS:1;f30636bb2128:40315 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-07T11:48:05,657 INFO [RS:0;f30636bb2128:38071 {}] regionserver.HRegionServer(3091): Received CLOSE for 85e3a3676af55f05d5f68bda5ac09869 2024-11-07T11:48:05,657 INFO [RS:1;f30636bb2128:40315 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;f30636bb2128:40315. 2024-11-07T11:48:05,657 DEBUG [RS:1;f30636bb2128:40315 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T11:48:05,657 DEBUG [RS:1;f30636bb2128:40315 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T11:48:05,658 INFO [RS:1;f30636bb2128:40315 {}] regionserver.HRegionServer(976): stopping server f30636bb2128,40315,1730980081104; all regions closed. 2024-11-07T11:48:05,658 INFO [RS:0;f30636bb2128:38071 {}] regionserver.HRegionServer(959): stopping server f30636bb2128,38071,1730980080953 2024-11-07T11:48:05,658 INFO [RS:0;f30636bb2128:38071 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-07T11:48:05,658 INFO [RS:0;f30636bb2128:38071 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;f30636bb2128:38071. 2024-11-07T11:48:05,658 DEBUG [RS:0;f30636bb2128:38071 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T11:48:05,658 DEBUG [RS:0;f30636bb2128:38071 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T11:48:05,658 INFO [RS:0;f30636bb2128:38071 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-07T11:48:05,658 INFO [RS:0;f30636bb2128:38071 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-07T11:48:05,658 INFO [RS:0;f30636bb2128:38071 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-07T11:48:05,658 INFO [RS:0;f30636bb2128:38071 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-07T11:48:05,658 INFO [RS_CLOSE_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1730980083247,,1730980083264.85e3a3676af55f05d5f68bda5ac09869. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-07T11:48:05,659 DEBUG [RS_CLOSE_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 85e3a3676af55f05d5f68bda5ac09869, disabling compactions & flushes 2024-11-07T11:48:05,659 INFO [RS:0;f30636bb2128:38071 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-07T11:48:05,659 INFO [RS_CLOSE_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1730980083247,,1730980083264.85e3a3676af55f05d5f68bda5ac09869. 2024-11-07T11:48:05,659 INFO [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-07T11:48:05,659 DEBUG [RS_CLOSE_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1730980083247,,1730980083264.85e3a3676af55f05d5f68bda5ac09869. 2024-11-07T11:48:05,659 DEBUG [RS:0;f30636bb2128:38071 {}] regionserver.HRegionServer(1325): Online Regions={85e3a3676af55f05d5f68bda5ac09869=testBlockEvictionOnRegionMove1730980083247,,1730980083264.85e3a3676af55f05d5f68bda5ac09869., 1588230740=hbase:meta,,1.1588230740} 2024-11-07T11:48:05,659 DEBUG [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-07T11:48:05,659 DEBUG [RS_CLOSE_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1730980083247,,1730980083264.85e3a3676af55f05d5f68bda5ac09869. after waiting 0 ms 2024-11-07T11:48:05,659 INFO [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-07T11:48:05,659 DEBUG [RS_CLOSE_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1730980083247,,1730980083264.85e3a3676af55f05d5f68bda5ac09869. 2024-11-07T11:48:05,659 DEBUG [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-07T11:48:05,659 DEBUG [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-07T11:48:05,659 DEBUG [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-07T11:48:05,659 DEBUG [RS:0;f30636bb2128:38071 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 85e3a3676af55f05d5f68bda5ac09869 2024-11-07T11:48:05,660 INFO [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-07T11:48:05,663 DEBUG [StoreFileCloser-85e3a3676af55f05d5f68bda5ac09869-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/default/testBlockEvictionOnRegionMove1730980083247/85e3a3676af55f05d5f68bda5ac09869/family/935687fdb97a471d9b6d0f2b6baf342d 2024-11-07T11:48:05,665 DEBUG [RS:1;f30636bb2128:40315 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/oldWALs 2024-11-07T11:48:05,665 INFO [RS:1;f30636bb2128:40315 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f30636bb2128%2C40315%2C1730980081104:(num 1730980082395) 2024-11-07T11:48:05,665 DEBUG [RS:1;f30636bb2128:40315 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T11:48:05,666 INFO [RS:1;f30636bb2128:40315 {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T11:48:05,666 INFO [RS:1;f30636bb2128:40315 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-07T11:48:05,666 INFO [RS:1;f30636bb2128:40315 {}] hbase.ChoreService(370): Chore service for: regionserver/f30636bb2128:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-07T11:48:05,666 INFO [RS:1;f30636bb2128:40315 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-07T11:48:05,666 INFO [RS:1;f30636bb2128:40315 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-07T11:48:05,666 INFO [RS:1;f30636bb2128:40315 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-07T11:48:05,666 INFO [regionserver/f30636bb2128:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-07T11:48:05,666 INFO [RS:1;f30636bb2128:40315 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-07T11:48:05,667 INFO [RS:1;f30636bb2128:40315 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40315 2024-11-07T11:48:05,669 DEBUG [RS_CLOSE_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/default/testBlockEvictionOnRegionMove1730980083247/85e3a3676af55f05d5f68bda5ac09869/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-07T11:48:05,670 INFO [RS_CLOSE_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1730980083247,,1730980083264.85e3a3676af55f05d5f68bda5ac09869. 2024-11-07T11:48:05,670 DEBUG [RS_CLOSE_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 85e3a3676af55f05d5f68bda5ac09869: Waiting for close lock at 1730980085658Running coprocessor pre-close hooks at 1730980085659 (+1 ms)Disabling compacts and flushes for region at 1730980085659Disabling writes for close at 1730980085659Writing region close event to WAL at 1730980085664 (+5 ms)Running coprocessor post-close hooks at 1730980085670 (+6 ms)Closed at 1730980085670 2024-11-07T11:48:05,670 DEBUG [RS_CLOSE_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1730980083247,,1730980083264.85e3a3676af55f05d5f68bda5ac09869. 2024-11-07T11:48:05,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40315-0x1001a11f38e0002, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f30636bb2128,40315,1730980081104 2024-11-07T11:48:05,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35319-0x1001a11f38e0000, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-07T11:48:05,671 INFO [RS:1;f30636bb2128:40315 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-07T11:48:05,674 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f30636bb2128,40315,1730980081104] 2024-11-07T11:48:05,675 DEBUG [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 32616366487a437882d523707777b218 2024-11-07T11:48:05,676 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f30636bb2128,40315,1730980081104 already deleted, retry=false 2024-11-07T11:48:05,676 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; f30636bb2128,40315,1730980081104 expired; onlineServers=1 2024-11-07T11:48:05,693 DEBUG [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740/.tmp/info/32616366487a437882d523707777b218 is 205, key is testBlockEvictionOnRegionMove1730980083247,,1730980083264.85e3a3676af55f05d5f68bda5ac09869./info:regioninfo/1730980085595/Put/seqid=0 2024-11-07T11:48:05,694 INFO [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740/.tmp/info/32616366487a437882d523707777b218 2024-11-07T11:48:05,702 DEBUG [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d6ae7665ccc94c8ea79ee79789a2a75e 2024-11-07T11:48:05,716 DEBUG [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740/.tmp/ns/d6ae7665ccc94c8ea79ee79789a2a75e is 43, key is default/ns:d/1730980083006/Put/seqid=0 2024-11-07T11:48:05,716 INFO [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740/.tmp/ns/d6ae7665ccc94c8ea79ee79789a2a75e 2024-11-07T11:48:05,719 INFO [regionserver/f30636bb2128:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T11:48:05,725 DEBUG [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 4a6422d1b47f4508becfeb7ee3d91606 2024-11-07T11:48:05,727 INFO [regionserver/f30636bb2128:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T11:48:05,737 DEBUG [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740/.tmp/table/4a6422d1b47f4508becfeb7ee3d91606 is 78, key is testBlockEvictionOnRegionMove1730980083247/table:state/1730980083683/Put/seqid=0 2024-11-07T11:48:05,737 INFO [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740/.tmp/table/4a6422d1b47f4508becfeb7ee3d91606 2024-11-07T11:48:05,740 DEBUG [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740/.tmp/info/32616366487a437882d523707777b218 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740/info/32616366487a437882d523707777b218 2024-11-07T11:48:05,741 DEBUG [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740/info/32616366487a437882d523707777b218, delay=924 ms 2024-11-07T11:48:05,742 INFO [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740/info/32616366487a437882d523707777b218, entries=15, sequenceid=15, filesize=7.6 K 2024-11-07T11:48:05,743 DEBUG [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740/.tmp/ns/d6ae7665ccc94c8ea79ee79789a2a75e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740/ns/d6ae7665ccc94c8ea79ee79789a2a75e 2024-11-07T11:48:05,744 DEBUG [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740/ns/d6ae7665ccc94c8ea79ee79789a2a75e, delay=982 ms 2024-11-07T11:48:05,745 INFO [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740/ns/d6ae7665ccc94c8ea79ee79789a2a75e, entries=2, sequenceid=15, filesize=5.0 K 2024-11-07T11:48:05,745 DEBUG [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740/.tmp/table/4a6422d1b47f4508becfeb7ee3d91606 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740/table/4a6422d1b47f4508becfeb7ee3d91606 2024-11-07T11:48:05,746 DEBUG [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740/table/4a6422d1b47f4508becfeb7ee3d91606, delay=987 ms 2024-11-07T11:48:05,747 INFO [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740/table/4a6422d1b47f4508becfeb7ee3d91606, entries=2, sequenceid=15, filesize=5.3 K 2024-11-07T11:48:05,748 INFO [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 89ms, sequenceid=15, compaction requested=false 2024-11-07T11:48:05,748 DEBUG [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-07T11:48:05,749 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740/info/32616366487a437882d523707777b218 2024-11-07T11:48:05,750 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740/ns/d6ae7665ccc94c8ea79ee79789a2a75e 2024-11-07T11:48:05,750 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740/table/4a6422d1b47f4508becfeb7ee3d91606 2024-11-07T11:48:05,755 DEBUG [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-07T11:48:05,755 DEBUG [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-07T11:48:05,756 INFO [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-07T11:48:05,756 DEBUG [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1730980085659Running coprocessor pre-close hooks at 1730980085659Disabling compacts and flushes for region at 1730980085659Disabling writes for close at 1730980085659Obtaining lock to block concurrent updates at 1730980085660 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1730980085660Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1730980085660Flushing stores of hbase:meta,,1.1588230740 at 1730980085660Flushing 1588230740/info: creating writer at 1730980085661 (+1 ms)Flushing 1588230740/info: appending metadata at 1730980085692 (+31 ms)Flushing 1588230740/info: closing flushed file at 1730980085692Flushing 1588230740/ns: creating writer at 1730980085696 (+4 ms)Flushing 1588230740/ns: appending metadata at 1730980085715 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1730980085715Flushing 1588230740/table: creating writer at 1730980085718 (+3 ms)Flushing 1588230740/table: appending metadata at 1730980085736 (+18 ms)Flushing 1588230740/table: closing flushed file at 1730980085736Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3fde295b: reopening flushed file at 1730980085740 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@45ea41c2: reopening flushed file at 1730980085742 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19f11a94: reopening flushed file at 1730980085745 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 89ms, sequenceid=15, compaction requested=false at 1730980085748 (+3 ms)Writing region close event to WAL at 1730980085751 (+3 ms)Running coprocessor post-close hooks at 1730980085755 (+4 ms)Closed at 1730980085756 (+1 ms) 2024-11-07T11:48:05,756 DEBUG [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-07T11:48:05,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40315-0x1001a11f38e0002, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T11:48:05,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40315-0x1001a11f38e0002, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T11:48:05,774 INFO [RS:1;f30636bb2128:40315 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-07T11:48:05,775 INFO [RS:1;f30636bb2128:40315 {}] regionserver.HRegionServer(1031): Exiting; stopping=f30636bb2128,40315,1730980081104; zookeeper connection closed. 2024-11-07T11:48:05,775 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@446163f0 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@446163f0 2024-11-07T11:48:05,860 INFO [RS:0;f30636bb2128:38071 {}] regionserver.HRegionServer(976): stopping server f30636bb2128,38071,1730980080953; all regions closed. 2024-11-07T11:48:05,862 DEBUG [RS:0;f30636bb2128:38071 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/oldWALs 2024-11-07T11:48:05,862 INFO [RS:0;f30636bb2128:38071 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f30636bb2128%2C38071%2C1730980080953.meta:.meta(num 1730980082865) 2024-11-07T11:48:05,864 DEBUG [RS:0;f30636bb2128:38071 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/oldWALs 2024-11-07T11:48:05,864 INFO [RS:0;f30636bb2128:38071 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f30636bb2128%2C38071%2C1730980080953:(num 1730980082395) 2024-11-07T11:48:05,864 DEBUG [RS:0;f30636bb2128:38071 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T11:48:05,864 INFO [RS:0;f30636bb2128:38071 {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T11:48:05,864 INFO [RS:0;f30636bb2128:38071 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-07T11:48:05,865 INFO [RS:0;f30636bb2128:38071 {}] hbase.ChoreService(370): Chore service for: regionserver/f30636bb2128:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-07T11:48:05,865 INFO [RS:0;f30636bb2128:38071 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-07T11:48:05,865 INFO [regionserver/f30636bb2128:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-07T11:48:05,865 INFO [RS:0;f30636bb2128:38071 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38071 2024-11-07T11:48:05,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35319-0x1001a11f38e0000, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-07T11:48:05,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38071-0x1001a11f38e0001, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f30636bb2128,38071,1730980080953 2024-11-07T11:48:05,868 INFO [RS:0;f30636bb2128:38071 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-07T11:48:05,870 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f30636bb2128,38071,1730980080953] 2024-11-07T11:48:05,871 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f30636bb2128,38071,1730980080953 already deleted, retry=false 2024-11-07T11:48:05,871 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; f30636bb2128,38071,1730980080953 expired; onlineServers=0 2024-11-07T11:48:05,872 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'f30636bb2128,35319,1730980080176' ***** 2024-11-07T11:48:05,872 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-07T11:48:05,872 INFO [M:0;f30636bb2128:35319 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-07T11:48:05,872 INFO [M:0;f30636bb2128:35319 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-07T11:48:05,872 DEBUG [M:0;f30636bb2128:35319 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-07T11:48:05,872 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-07T11:48:05,872 DEBUG [M:0;f30636bb2128:35319 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-07T11:48:05,872 DEBUG [master/f30636bb2128:0:becomeActiveMaster-HFileCleaner.small.0-1730980082095 {}] cleaner.HFileCleaner(306): Exit Thread[master/f30636bb2128:0:becomeActiveMaster-HFileCleaner.small.0-1730980082095,5,FailOnTimeoutGroup] 2024-11-07T11:48:05,872 DEBUG [master/f30636bb2128:0:becomeActiveMaster-HFileCleaner.large.0-1730980082094 {}] cleaner.HFileCleaner(306): Exit Thread[master/f30636bb2128:0:becomeActiveMaster-HFileCleaner.large.0-1730980082094,5,FailOnTimeoutGroup] 2024-11-07T11:48:05,873 INFO [M:0;f30636bb2128:35319 {}] hbase.ChoreService(370): Chore service for: master/f30636bb2128:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-07T11:48:05,873 INFO [M:0;f30636bb2128:35319 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-07T11:48:05,873 DEBUG [M:0;f30636bb2128:35319 {}] master.HMaster(1795): Stopping service threads 2024-11-07T11:48:05,873 INFO [M:0;f30636bb2128:35319 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-07T11:48:05,873 INFO [M:0;f30636bb2128:35319 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T11:48:05,874 INFO [M:0;f30636bb2128:35319 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T11:48:05,874 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-07T11:48:05,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35319-0x1001a11f38e0000, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-07T11:48:05,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35319-0x1001a11f38e0000, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:48:05,875 DEBUG [M:0;f30636bb2128:35319 {}] zookeeper.ZKUtil(347): master:35319-0x1001a11f38e0000, quorum=127.0.0.1:64253, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-07T11:48:05,875 WARN [M:0;f30636bb2128:35319 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-07T11:48:05,875 INFO [M:0;f30636bb2128:35319 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/.lastflushedseqids 2024-11-07T11:48:05,888 INFO [M:0;f30636bb2128:35319 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T11:48:05,888 INFO [M:0;f30636bb2128:35319 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-07T11:48:05,888 DEBUG [M:0;f30636bb2128:35319 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-07T11:48:05,888 INFO [M:0;f30636bb2128:35319 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T11:48:05,888 DEBUG [M:0;f30636bb2128:35319 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T11:48:05,888 DEBUG [M:0;f30636bb2128:35319 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-07T11:48:05,888 DEBUG [M:0;f30636bb2128:35319 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T11:48:05,889 INFO [M:0;f30636bb2128:35319 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.67 KB heapSize=43.02 KB 2024-11-07T11:48:05,898 DEBUG [M:0;f30636bb2128:35319 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 0f86c3ac74df4eee97eee6789fc77f6d 2024-11-07T11:48:05,912 DEBUG [M:0;f30636bb2128:35319 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0f86c3ac74df4eee97eee6789fc77f6d is 82, key is hbase:meta,,1/info:regioninfo/1730980082936/Put/seqid=0 2024-11-07T11:48:05,913 INFO [M:0;f30636bb2128:35319 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0f86c3ac74df4eee97eee6789fc77f6d 2024-11-07T11:48:05,921 DEBUG [M:0;f30636bb2128:35319 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 96b3cdeba335496a9fc00c0c4e118c77 2024-11-07T11:48:05,935 DEBUG [M:0;f30636bb2128:35319 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/96b3cdeba335496a9fc00c0c4e118c77 is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1730980083689/Put/seqid=0 2024-11-07T11:48:05,936 INFO [M:0;f30636bb2128:35319 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.05 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/96b3cdeba335496a9fc00c0c4e118c77 2024-11-07T11:48:05,943 DEBUG [M:0;f30636bb2128:35319 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on de9dab1ece0a43ce95e0eb787ab8d8e4 2024-11-07T11:48:05,956 DEBUG [M:0;f30636bb2128:35319 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/de9dab1ece0a43ce95e0eb787ab8d8e4 is 69, key is f30636bb2128,38071,1730980080953/rs:state/1730980082106/Put/seqid=0 2024-11-07T11:48:05,956 INFO [M:0;f30636bb2128:35319 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/de9dab1ece0a43ce95e0eb787ab8d8e4 2024-11-07T11:48:05,958 DEBUG [M:0;f30636bb2128:35319 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0f86c3ac74df4eee97eee6789fc77f6d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/0f86c3ac74df4eee97eee6789fc77f6d 2024-11-07T11:48:05,960 INFO [M:0;f30636bb2128:35319 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/0f86c3ac74df4eee97eee6789fc77f6d, entries=8, sequenceid=84, filesize=5.5 K 2024-11-07T11:48:05,960 DEBUG [M:0;f30636bb2128:35319 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/96b3cdeba335496a9fc00c0c4e118c77 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/96b3cdeba335496a9fc00c0c4e118c77 2024-11-07T11:48:05,962 INFO [M:0;f30636bb2128:35319 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/96b3cdeba335496a9fc00c0c4e118c77, entries=9, sequenceid=84, filesize=6.5 K 2024-11-07T11:48:05,962 DEBUG [M:0;f30636bb2128:35319 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/de9dab1ece0a43ce95e0eb787ab8d8e4 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/de9dab1ece0a43ce95e0eb787ab8d8e4 2024-11-07T11:48:05,963 INFO [M:0;f30636bb2128:35319 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/de9dab1ece0a43ce95e0eb787ab8d8e4, entries=2, sequenceid=84, filesize=5.1 K 2024-11-07T11:48:05,964 INFO [M:0;f30636bb2128:35319 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 76ms, sequenceid=84, compaction requested=false 2024-11-07T11:48:05,966 INFO [M:0;f30636bb2128:35319 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T11:48:05,966 DEBUG [M:0;f30636bb2128:35319 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1730980085888Disabling compacts and flushes for region at 1730980085888Disabling writes for close at 1730980085888Obtaining lock to block concurrent updates at 1730980085889 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1730980085889Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35506, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1730980085889Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1730980085889Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1730980085890 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1730980085912 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1730980085912Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1730980085915 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1730980085935 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1730980085935Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1730980085937 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1730980085955 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1730980085955Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77d0bd63: reopening flushed file at 1730980085958 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37959cef: reopening flushed file at 1730980085960 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13452a60: reopening flushed file at 1730980085962 (+2 ms)Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 76ms, sequenceid=84, compaction requested=false at 1730980085964 (+2 ms)Writing region close event to WAL at 1730980085966 (+2 ms)Closed at 1730980085966 2024-11-07T11:48:05,968 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-07T11:48:05,968 INFO [M:0;f30636bb2128:35319 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-07T11:48:05,969 INFO [M:0;f30636bb2128:35319 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35319 2024-11-07T11:48:05,969 INFO [M:0;f30636bb2128:35319 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-07T11:48:05,970 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38071-0x1001a11f38e0001, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T11:48:05,970 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38071-0x1001a11f38e0001, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T11:48:05,970 INFO [RS:0;f30636bb2128:38071 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-07T11:48:05,970 INFO [RS:0;f30636bb2128:38071 {}] regionserver.HRegionServer(1031): Exiting; stopping=f30636bb2128,38071,1730980080953; zookeeper connection closed. 2024-11-07T11:48:05,970 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3ab41963 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3ab41963 2024-11-07T11:48:05,971 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-07T11:48:06,071 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35319-0x1001a11f38e0000, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T11:48:06,071 INFO [M:0;f30636bb2128:35319 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-07T11:48:06,071 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35319-0x1001a11f38e0000, quorum=127.0.0.1:64253, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T11:48:06,076 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-07T11:48:06,096 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-07T11:48:06,096 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T11:48:06,104 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=51 (was 12) Potentially hanging thread: regionserver/f30636bb2128:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/f30636bb2128:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1730980085744 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1730980084051 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1730980085582 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: regionserver/f30636bb2128:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/f30636bb2128:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1730980085741 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1905 (was 2206), ProcessCount=11 (was 11), AvailableMemoryMB=2919 (was 3279) 2024-11-07T11:48:06,110 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=51, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=1905, ProcessCount=11, AvailableMemoryMB=2919 2024-11-07T11:48:06,110 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/hadoop-log-dir so I do NOT create it in target/test-data/e5f86774-a615-bd53-b3f4-8c9383f47bad 2024-11-07T11:48:06,110 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/hadoop-tmp-dir so I do NOT create it in target/test-data/e5f86774-a615-bd53-b3f4-8c9383f47bad 2024-11-07T11:48:06,110 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T11:48:06,113 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5f86774-a615-bd53-b3f4-8c9383f47bad/cluster_a0ac5ac7-eed0-1a2e-640f-6b6593f6180f, deleteOnExit=true 2024-11-07T11:48:06,116 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5f86774-a615-bd53-b3f4-8c9383f47bad/cluster_a0ac5ac7-eed0-1a2e-640f-6b6593f6180f/zookeeper_0, clientPort=63417, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5f86774-a615-bd53-b3f4-8c9383f47bad/cluster_a0ac5ac7-eed0-1a2e-640f-6b6593f6180f/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5f86774-a615-bd53-b3f4-8c9383f47bad/cluster_a0ac5ac7-eed0-1a2e-640f-6b6593f6180f/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-07T11:48:06,117 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63417 2024-11-07T11:48:06,128 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47 with version=8 2024-11-07T11:48:06,128 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/hbase-staging 2024-11-07T11:48:06,131 INFO [Time-limited test {}] client.ConnectionUtils(128): master/f30636bb2128:0 server-side Connection retries=45 2024-11-07T11:48:06,131 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T11:48:06,131 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-07T11:48:06,131 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-07T11:48:06,131 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T11:48:06,131 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-07T11:48:06,131 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-07T11:48:06,131 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-07T11:48:06,132 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35521 2024-11-07T11:48:06,134 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35521 connecting to ZooKeeper ensemble=127.0.0.1:63417 2024-11-07T11:48:06,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:355210x0, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-07T11:48:06,140 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35521-0x1001a120c020000 connected 2024-11-07T11:48:06,159 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T11:48:06,160 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T11:48:06,160 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35521-0x1001a120c020000, quorum=127.0.0.1:63417, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T11:48:06,161 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47, hbase.cluster.distributed=false 2024-11-07T11:48:06,162 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35521-0x1001a120c020000, quorum=127.0.0.1:63417, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-07T11:48:06,165 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35521 2024-11-07T11:48:06,168 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35521 2024-11-07T11:48:06,169 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35521 2024-11-07T11:48:06,171 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35521 2024-11-07T11:48:06,172 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35521 2024-11-07T11:48:06,189 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f30636bb2128:0 server-side Connection retries=45 2024-11-07T11:48:06,189 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T11:48:06,189 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-07T11:48:06,189 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-07T11:48:06,189 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T11:48:06,189 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-07T11:48:06,189 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-07T11:48:06,189 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-07T11:48:06,190 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41943 2024-11-07T11:48:06,191 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41943 connecting to ZooKeeper ensemble=127.0.0.1:63417 2024-11-07T11:48:06,192 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T11:48:06,192 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T11:48:06,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:419430x0, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-07T11:48:06,195 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41943-0x1001a120c020001 connected 2024-11-07T11:48:06,195 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41943-0x1001a120c020001, quorum=127.0.0.1:63417, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T11:48:06,195 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-07T11:48:06,196 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5f86774-a615-bd53-b3f4-8c9383f47bad/bucket.cache 2024-11-07T11:48:06,196 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-07T11:48:06,196 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-07T11:48:06,196 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-07T11:48:06,197 INFO [Thread-121 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-07T11:48:06,197 WARN [Thread-121 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-07T11:48:06,197 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5f86774-a615-bd53-b3f4-8c9383f47bad/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5f86774-a615-bd53-b3f4-8c9383f47bad/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-07T11:48:06,197 INFO [Thread-121 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-07T11:48:06,197 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-07T11:48:06,197 INFO [Thread-121 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5f86774-a615-bd53-b3f4-8c9383f47bad/bucket.persistence is complete. 2024-11-07T11:48:06,198 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41943-0x1001a120c020001, quorum=127.0.0.1:63417, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-07T11:48:06,200 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41943-0x1001a120c020001, quorum=127.0.0.1:63417, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-07T11:48:06,200 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41943 2024-11-07T11:48:06,200 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41943 2024-11-07T11:48:06,201 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41943 2024-11-07T11:48:06,201 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41943 2024-11-07T11:48:06,201 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41943 2024-11-07T11:48:06,216 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f30636bb2128:0 server-side Connection retries=45 2024-11-07T11:48:06,216 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T11:48:06,216 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-07T11:48:06,217 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-07T11:48:06,217 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T11:48:06,217 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-07T11:48:06,217 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-07T11:48:06,217 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-07T11:48:06,218 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33625 2024-11-07T11:48:06,219 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33625 connecting to ZooKeeper ensemble=127.0.0.1:63417 2024-11-07T11:48:06,219 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T11:48:06,219 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T11:48:06,223 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:336250x0, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-07T11:48:06,223 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33625-0x1001a120c020002 connected 2024-11-07T11:48:06,223 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33625-0x1001a120c020002, quorum=127.0.0.1:63417, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T11:48:06,223 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-07T11:48:06,224 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5f86774-a615-bd53-b3f4-8c9383f47bad/bucket.cache 2024-11-07T11:48:06,224 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-07T11:48:06,225 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-07T11:48:06,225 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-07T11:48:06,225 INFO [Thread-122 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-07T11:48:06,225 WARN [Thread-122 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-07T11:48:06,225 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5f86774-a615-bd53-b3f4-8c9383f47bad/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5f86774-a615-bd53-b3f4-8c9383f47bad/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-07T11:48:06,226 INFO [Thread-122 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-07T11:48:06,226 INFO [Thread-122 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5f86774-a615-bd53-b3f4-8c9383f47bad/bucket.persistence is complete. 2024-11-07T11:48:06,226 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-07T11:48:06,227 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33625-0x1001a120c020002, quorum=127.0.0.1:63417, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-07T11:48:06,228 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33625-0x1001a120c020002, quorum=127.0.0.1:63417, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-07T11:48:06,232 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33625 2024-11-07T11:48:06,233 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33625 2024-11-07T11:48:06,233 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33625 2024-11-07T11:48:06,233 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33625 2024-11-07T11:48:06,234 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33625 2024-11-07T11:48:06,246 DEBUG [M:0;f30636bb2128:35521 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;f30636bb2128:35521 2024-11-07T11:48:06,246 INFO [master/f30636bb2128:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/f30636bb2128,35521,1730980086130 2024-11-07T11:48:06,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33625-0x1001a120c020002, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T11:48:06,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35521-0x1001a120c020000, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T11:48:06,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41943-0x1001a120c020001, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T11:48:06,249 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35521-0x1001a120c020000, quorum=127.0.0.1:63417, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/f30636bb2128,35521,1730980086130 2024-11-07T11:48:06,251 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33625-0x1001a120c020002, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-07T11:48:06,251 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35521-0x1001a120c020000, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:48:06,251 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41943-0x1001a120c020001, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-07T11:48:06,251 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33625-0x1001a120c020002, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:48:06,251 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41943-0x1001a120c020001, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:48:06,252 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35521-0x1001a120c020000, quorum=127.0.0.1:63417, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-07T11:48:06,253 INFO [master/f30636bb2128:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/f30636bb2128,35521,1730980086130 from backup master directory 2024-11-07T11:48:06,254 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33625-0x1001a120c020002, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T11:48:06,255 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41943-0x1001a120c020001, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T11:48:06,255 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35521-0x1001a120c020000, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/f30636bb2128,35521,1730980086130 2024-11-07T11:48:06,255 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35521-0x1001a120c020000, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T11:48:06,255 WARN [master/f30636bb2128:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-07T11:48:06,255 INFO [master/f30636bb2128:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=f30636bb2128,35521,1730980086130 2024-11-07T11:48:06,256 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/hbase.id] with ID: 2ec45d0e-1fd0-4ed6-b48a-056cb9ebaeee 2024-11-07T11:48:06,256 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/.tmp/hbase.id 2024-11-07T11:48:06,261 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/hbase.id] 2024-11-07T11:48:06,306 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T11:48:06,306 INFO [master/f30636bb2128:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-07T11:48:06,307 INFO [master/f30636bb2128:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-11-07T11:48:06,310 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33625-0x1001a120c020002, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:48:06,310 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41943-0x1001a120c020001, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:48:06,310 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35521-0x1001a120c020000, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:48:06,315 INFO [master/f30636bb2128:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T11:48:06,316 INFO [master/f30636bb2128:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-07T11:48:06,316 INFO [master/f30636bb2128:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T11:48:06,337 INFO [master/f30636bb2128:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/MasterData/data/master/store 2024-11-07T11:48:06,343 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:48:06,344 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-07T11:48:06,344 INFO [master/f30636bb2128:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T11:48:06,344 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T11:48:06,344 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-07T11:48:06,344 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T11:48:06,344 INFO [master/f30636bb2128:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T11:48:06,344 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1730980086343Disabling compacts and flushes for region at 1730980086343Disabling writes for close at 1730980086344 (+1 ms)Writing region close event to WAL at 1730980086344Closed at 1730980086344 2024-11-07T11:48:06,344 WARN [master/f30636bb2128:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/MasterData/data/master/store/.initializing 2024-11-07T11:48:06,344 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/MasterData/WALs/f30636bb2128,35521,1730980086130 2024-11-07T11:48:06,350 INFO [master/f30636bb2128:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-07T11:48:06,353 INFO [master/f30636bb2128:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f30636bb2128%2C35521%2C1730980086130, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/MasterData/WALs/f30636bb2128,35521,1730980086130, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/MasterData/oldWALs, maxLogs=10 2024-11-07T11:48:06,371 INFO [master/f30636bb2128:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/MasterData/WALs/f30636bb2128,35521,1730980086130/f30636bb2128%2C35521%2C1730980086130.1730980086353 2024-11-07T11:48:06,371 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-07T11:48:06,371 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-07T11:48:06,371 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:48:06,371 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T11:48:06,372 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T11:48:06,372 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T11:48:06,376 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-07T11:48:06,377 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:48:06,377 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T11:48:06,377 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T11:48:06,380 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-07T11:48:06,380 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:48:06,380 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T11:48:06,380 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T11:48:06,384 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-07T11:48:06,384 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:48:06,384 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T11:48:06,384 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T11:48:06,387 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-07T11:48:06,387 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:48:06,387 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T11:48:06,388 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T11:48:06,388 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-07T11:48:06,388 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-07T11:48:06,388 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T11:48:06,388 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T11:48:06,389 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-07T11:48:06,389 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T11:48:06,395 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T11:48:06,395 INFO [master/f30636bb2128:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62494585, jitterRate=-0.06875811517238617}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-07T11:48:06,396 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1730980086372Initializing all the Stores at 1730980086372Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730980086372Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730980086372Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730980086372Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730980086372Cleaning up temporary data from old regions at 1730980086388 (+16 ms)Region opened successfully at 1730980086396 (+8 ms) 2024-11-07T11:48:06,397 INFO [master/f30636bb2128:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-07T11:48:06,401 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26cf6101, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f30636bb2128/172.17.0.2:0 2024-11-07T11:48:06,402 INFO [master/f30636bb2128:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-07T11:48:06,402 INFO [master/f30636bb2128:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-07T11:48:06,402 INFO [master/f30636bb2128:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-07T11:48:06,402 INFO [master/f30636bb2128:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T11:48:06,403 INFO [master/f30636bb2128:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T11:48:06,403 INFO [master/f30636bb2128:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-07T11:48:06,403 INFO [master/f30636bb2128:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-07T11:48:06,405 INFO [master/f30636bb2128:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-07T11:48:06,406 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35521-0x1001a120c020000, quorum=127.0.0.1:63417, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-07T11:48:06,407 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-07T11:48:06,407 INFO [master/f30636bb2128:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-07T11:48:06,408 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35521-0x1001a120c020000, quorum=127.0.0.1:63417, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-07T11:48:06,409 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-07T11:48:06,409 INFO [master/f30636bb2128:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-07T11:48:06,410 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35521-0x1001a120c020000, quorum=127.0.0.1:63417, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-07T11:48:06,412 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-07T11:48:06,412 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35521-0x1001a120c020000, quorum=127.0.0.1:63417, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-07T11:48:06,414 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-07T11:48:06,416 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35521-0x1001a120c020000, quorum=127.0.0.1:63417, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-07T11:48:06,417 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-07T11:48:06,419 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33625-0x1001a120c020002, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-07T11:48:06,419 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35521-0x1001a120c020000, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-07T11:48:06,419 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41943-0x1001a120c020001, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-07T11:48:06,419 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33625-0x1001a120c020002, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:48:06,419 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35521-0x1001a120c020000, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:48:06,419 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41943-0x1001a120c020001, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:48:06,420 INFO [master/f30636bb2128:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=f30636bb2128,35521,1730980086130, sessionid=0x1001a120c020000, setting cluster-up flag (Was=false) 2024-11-07T11:48:06,423 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35521-0x1001a120c020000, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:48:06,423 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33625-0x1001a120c020002, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:48:06,423 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41943-0x1001a120c020001, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:48:06,428 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-07T11:48:06,430 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f30636bb2128,35521,1730980086130 2024-11-07T11:48:06,432 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33625-0x1001a120c020002, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:48:06,432 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35521-0x1001a120c020000, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:48:06,432 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41943-0x1001a120c020001, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:48:06,438 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-07T11:48:06,439 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f30636bb2128,35521,1730980086130 2024-11-07T11:48:06,440 INFO [master/f30636bb2128:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-07T11:48:06,441 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T11:48:06,442 INFO [master/f30636bb2128:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-07T11:48:06,442 INFO [master/f30636bb2128:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-07T11:48:06,442 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: f30636bb2128,35521,1730980086130 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-07T11:48:06,444 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/f30636bb2128:0, corePoolSize=5, maxPoolSize=5 2024-11-07T11:48:06,444 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/f30636bb2128:0, corePoolSize=5, maxPoolSize=5 2024-11-07T11:48:06,444 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/f30636bb2128:0, corePoolSize=5, maxPoolSize=5 2024-11-07T11:48:06,444 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/f30636bb2128:0, corePoolSize=5, maxPoolSize=5 2024-11-07T11:48:06,444 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/f30636bb2128:0, corePoolSize=10, maxPoolSize=10 2024-11-07T11:48:06,444 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:06,444 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/f30636bb2128:0, corePoolSize=2, maxPoolSize=2 2024-11-07T11:48:06,444 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:06,445 INFO [master/f30636bb2128:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1730980116445 2024-11-07T11:48:06,445 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-07T11:48:06,446 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-07T11:48:06,446 INFO [master/f30636bb2128:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-07T11:48:06,446 INFO [master/f30636bb2128:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-07T11:48:06,446 INFO [master/f30636bb2128:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-07T11:48:06,446 INFO [master/f30636bb2128:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-07T11:48:06,446 INFO [master/f30636bb2128:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-07T11:48:06,446 INFO [master/f30636bb2128:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-07T11:48:06,446 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:48:06,446 INFO [master/f30636bb2128:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:06,446 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-07T11:48:06,447 INFO [master/f30636bb2128:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-07T11:48:06,447 INFO [master/f30636bb2128:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-07T11:48:06,447 INFO [master/f30636bb2128:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-07T11:48:06,447 INFO [master/f30636bb2128:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-07T11:48:06,448 INFO [master/f30636bb2128:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-07T11:48:06,448 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/f30636bb2128:0:becomeActiveMaster-HFileCleaner.large.0-1730980086448,5,FailOnTimeoutGroup] 2024-11-07T11:48:06,448 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/f30636bb2128:0:becomeActiveMaster-HFileCleaner.small.0-1730980086448,5,FailOnTimeoutGroup] 2024-11-07T11:48:06,448 INFO [master/f30636bb2128:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:06,448 INFO [master/f30636bb2128:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-07T11:48:06,448 INFO [master/f30636bb2128:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:06,448 INFO [master/f30636bb2128:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:06,458 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-07T11:48:06,458 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47 2024-11-07T11:48:06,464 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:48:06,465 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-07T11:48:06,468 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-07T11:48:06,468 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:48:06,469 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T11:48:06,469 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-07T11:48:06,472 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-07T11:48:06,472 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:48:06,472 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T11:48:06,472 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-07T11:48:06,476 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-07T11:48:06,476 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:48:06,476 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T11:48:06,476 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-07T11:48:06,480 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-07T11:48:06,480 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:48:06,480 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T11:48:06,480 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-07T11:48:06,481 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740 2024-11-07T11:48:06,481 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740 2024-11-07T11:48:06,481 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-07T11:48:06,481 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-07T11:48:06,481 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-07T11:48:06,482 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-07T11:48:06,488 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T11:48:06,488 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71359576, jitterRate=0.06334054470062256}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-07T11:48:06,490 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1730980086464Initializing all the Stores at 1730980086465 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730980086465Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730980086465Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730980086465Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730980086465Cleaning up temporary data from old regions at 1730980086481 (+16 ms)Region opened successfully at 1730980086489 (+8 ms) 2024-11-07T11:48:06,490 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-07T11:48:06,490 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-07T11:48:06,490 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-07T11:48:06,490 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-07T11:48:06,490 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-07T11:48:06,490 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-07T11:48:06,490 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1730980086490Disabling compacts and flushes for region at 1730980086490Disabling writes for close at 1730980086490Writing region close event to WAL at 1730980086490Closed at 1730980086490 2024-11-07T11:48:06,491 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-07T11:48:06,491 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-07T11:48:06,491 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-07T11:48:06,492 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T11:48:06,493 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-07T11:48:06,536 INFO [RS:1;f30636bb2128:33625 {}] regionserver.HRegionServer(746): ClusterId : 2ec45d0e-1fd0-4ed6-b48a-056cb9ebaeee 2024-11-07T11:48:06,536 INFO [RS:0;f30636bb2128:41943 {}] regionserver.HRegionServer(746): ClusterId : 2ec45d0e-1fd0-4ed6-b48a-056cb9ebaeee 2024-11-07T11:48:06,537 DEBUG [RS:0;f30636bb2128:41943 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-07T11:48:06,537 DEBUG [RS:1;f30636bb2128:33625 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-07T11:48:06,539 DEBUG [RS:0;f30636bb2128:41943 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-07T11:48:06,539 DEBUG [RS:1;f30636bb2128:33625 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-07T11:48:06,539 DEBUG [RS:0;f30636bb2128:41943 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-07T11:48:06,539 DEBUG [RS:1;f30636bb2128:33625 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-07T11:48:06,542 DEBUG [RS:0;f30636bb2128:41943 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-07T11:48:06,542 DEBUG [RS:1;f30636bb2128:33625 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-07T11:48:06,542 DEBUG [RS:0;f30636bb2128:41943 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66aeeeaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f30636bb2128/172.17.0.2:0 2024-11-07T11:48:06,542 DEBUG [RS:1;f30636bb2128:33625 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c3d5747, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f30636bb2128/172.17.0.2:0 2024-11-07T11:48:06,555 DEBUG [RS:0;f30636bb2128:41943 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;f30636bb2128:41943 2024-11-07T11:48:06,556 INFO [RS:0;f30636bb2128:41943 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-07T11:48:06,556 INFO [RS:0;f30636bb2128:41943 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-07T11:48:06,556 DEBUG [RS:0;f30636bb2128:41943 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-07T11:48:06,556 INFO [RS:0;f30636bb2128:41943 {}] regionserver.HRegionServer(2659): reportForDuty to master=f30636bb2128,35521,1730980086130 with port=41943, startcode=1730980086188 2024-11-07T11:48:06,557 DEBUG [RS:0;f30636bb2128:41943 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-07T11:48:06,559 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53927, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-07T11:48:06,560 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35521 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f30636bb2128,41943,1730980086188 2024-11-07T11:48:06,560 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35521 {}] master.ServerManager(517): Registering regionserver=f30636bb2128,41943,1730980086188 2024-11-07T11:48:06,561 DEBUG [RS:1;f30636bb2128:33625 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;f30636bb2128:33625 2024-11-07T11:48:06,561 INFO [RS:1;f30636bb2128:33625 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-07T11:48:06,561 INFO [RS:1;f30636bb2128:33625 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-07T11:48:06,561 DEBUG [RS:1;f30636bb2128:33625 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-07T11:48:06,562 DEBUG [RS:0;f30636bb2128:41943 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47 2024-11-07T11:48:06,562 DEBUG [RS:0;f30636bb2128:41943 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-07T11:48:06,562 DEBUG [RS:0;f30636bb2128:41943 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-07T11:48:06,563 INFO [RS:1;f30636bb2128:33625 {}] regionserver.HRegionServer(2659): reportForDuty to master=f30636bb2128,35521,1730980086130 with port=33625, startcode=1730980086216 2024-11-07T11:48:06,563 DEBUG [RS:1;f30636bb2128:33625 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-07T11:48:06,565 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35521-0x1001a120c020000, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-07T11:48:06,565 DEBUG [RS:0;f30636bb2128:41943 {}] zookeeper.ZKUtil(111): regionserver:41943-0x1001a120c020001, quorum=127.0.0.1:63417, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f30636bb2128,41943,1730980086188 2024-11-07T11:48:06,566 WARN [RS:0;f30636bb2128:41943 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-07T11:48:06,566 INFO [RS:0;f30636bb2128:41943 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T11:48:06,566 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36771, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-07T11:48:06,566 DEBUG [RS:0;f30636bb2128:41943 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/WALs/f30636bb2128,41943,1730980086188 2024-11-07T11:48:06,566 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f30636bb2128,41943,1730980086188] 2024-11-07T11:48:06,566 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35521 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f30636bb2128,33625,1730980086216 2024-11-07T11:48:06,567 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35521 {}] master.ServerManager(517): Registering regionserver=f30636bb2128,33625,1730980086216 2024-11-07T11:48:06,568 DEBUG [RS:1;f30636bb2128:33625 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47 2024-11-07T11:48:06,568 DEBUG [RS:1;f30636bb2128:33625 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-07T11:48:06,569 DEBUG [RS:1;f30636bb2128:33625 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-07T11:48:06,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35521-0x1001a120c020000, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-07T11:48:06,575 DEBUG [RS:1;f30636bb2128:33625 {}] zookeeper.ZKUtil(111): regionserver:33625-0x1001a120c020002, quorum=127.0.0.1:63417, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f30636bb2128,33625,1730980086216 2024-11-07T11:48:06,575 WARN [RS:1;f30636bb2128:33625 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-07T11:48:06,575 INFO [RS:1;f30636bb2128:33625 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T11:48:06,575 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f30636bb2128,33625,1730980086216] 2024-11-07T11:48:06,575 DEBUG [RS:1;f30636bb2128:33625 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/WALs/f30636bb2128,33625,1730980086216 2024-11-07T11:48:06,576 INFO [RS:0;f30636bb2128:41943 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-07T11:48:06,579 INFO [RS:0;f30636bb2128:41943 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-07T11:48:06,579 INFO [RS:0;f30636bb2128:41943 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-07T11:48:06,580 INFO [RS:0;f30636bb2128:41943 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:06,580 INFO [RS:0;f30636bb2128:41943 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-07T11:48:06,581 INFO [RS:0;f30636bb2128:41943 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-07T11:48:06,581 INFO [RS:0;f30636bb2128:41943 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:06,581 DEBUG [RS:0;f30636bb2128:41943 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:06,581 DEBUG [RS:0;f30636bb2128:41943 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:06,581 DEBUG [RS:0;f30636bb2128:41943 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:06,581 DEBUG [RS:0;f30636bb2128:41943 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:06,582 DEBUG [RS:0;f30636bb2128:41943 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:06,582 DEBUG [RS:0;f30636bb2128:41943 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f30636bb2128:0, corePoolSize=2, maxPoolSize=2 2024-11-07T11:48:06,582 DEBUG [RS:0;f30636bb2128:41943 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:06,582 DEBUG [RS:0;f30636bb2128:41943 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:06,582 INFO [RS:1;f30636bb2128:33625 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-07T11:48:06,582 DEBUG [RS:0;f30636bb2128:41943 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:06,582 DEBUG [RS:0;f30636bb2128:41943 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:06,582 DEBUG [RS:0;f30636bb2128:41943 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:06,582 DEBUG [RS:0;f30636bb2128:41943 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:06,583 DEBUG [RS:0;f30636bb2128:41943 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f30636bb2128:0, corePoolSize=3, maxPoolSize=3 2024-11-07T11:48:06,583 DEBUG [RS:0;f30636bb2128:41943 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f30636bb2128:0, corePoolSize=3, maxPoolSize=3 2024-11-07T11:48:06,585 INFO [RS:1;f30636bb2128:33625 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-07T11:48:06,585 INFO [RS:0;f30636bb2128:41943 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:06,585 INFO [RS:0;f30636bb2128:41943 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:06,585 INFO [RS:0;f30636bb2128:41943 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:06,585 INFO [RS:1;f30636bb2128:33625 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-07T11:48:06,585 INFO [RS:0;f30636bb2128:41943 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:06,585 INFO [RS:1;f30636bb2128:33625 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:06,585 INFO [RS:0;f30636bb2128:41943 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:06,585 INFO [RS:0;f30636bb2128:41943 {}] hbase.ChoreService(168): Chore ScheduledChore name=f30636bb2128,41943,1730980086188-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-07T11:48:06,585 INFO [RS:1;f30636bb2128:33625 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-07T11:48:06,586 INFO [RS:1;f30636bb2128:33625 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-07T11:48:06,586 INFO [RS:1;f30636bb2128:33625 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:06,586 DEBUG [RS:1;f30636bb2128:33625 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:06,586 DEBUG [RS:1;f30636bb2128:33625 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:06,586 DEBUG [RS:1;f30636bb2128:33625 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:06,586 DEBUG [RS:1;f30636bb2128:33625 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:06,586 DEBUG [RS:1;f30636bb2128:33625 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:06,587 DEBUG [RS:1;f30636bb2128:33625 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f30636bb2128:0, corePoolSize=2, maxPoolSize=2 2024-11-07T11:48:06,587 DEBUG [RS:1;f30636bb2128:33625 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:06,587 DEBUG [RS:1;f30636bb2128:33625 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:06,587 DEBUG [RS:1;f30636bb2128:33625 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:06,587 DEBUG [RS:1;f30636bb2128:33625 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:06,587 DEBUG [RS:1;f30636bb2128:33625 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:06,587 DEBUG [RS:1;f30636bb2128:33625 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:06,587 DEBUG [RS:1;f30636bb2128:33625 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f30636bb2128:0, corePoolSize=3, maxPoolSize=3 2024-11-07T11:48:06,587 DEBUG [RS:1;f30636bb2128:33625 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f30636bb2128:0, corePoolSize=3, maxPoolSize=3 2024-11-07T11:48:06,591 INFO [RS:1;f30636bb2128:33625 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:06,591 INFO [RS:1;f30636bb2128:33625 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:06,591 INFO [RS:1;f30636bb2128:33625 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:06,591 INFO [RS:1;f30636bb2128:33625 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:06,591 INFO [RS:1;f30636bb2128:33625 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:06,591 INFO [RS:1;f30636bb2128:33625 {}] hbase.ChoreService(168): Chore ScheduledChore name=f30636bb2128,33625,1730980086216-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-07T11:48:06,606 INFO [RS:0;f30636bb2128:41943 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-07T11:48:06,606 INFO [RS:0;f30636bb2128:41943 {}] hbase.ChoreService(168): Chore ScheduledChore name=f30636bb2128,41943,1730980086188-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:06,606 INFO [RS:0;f30636bb2128:41943 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:06,606 INFO [RS:0;f30636bb2128:41943 {}] regionserver.Replication(171): f30636bb2128,41943,1730980086188 started 2024-11-07T11:48:06,607 INFO [RS:1;f30636bb2128:33625 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-07T11:48:06,607 INFO [RS:1;f30636bb2128:33625 {}] hbase.ChoreService(168): Chore ScheduledChore name=f30636bb2128,33625,1730980086216-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:06,608 INFO [RS:1;f30636bb2128:33625 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:06,608 INFO [RS:1;f30636bb2128:33625 {}] regionserver.Replication(171): f30636bb2128,33625,1730980086216 started 2024-11-07T11:48:06,622 INFO [RS:1;f30636bb2128:33625 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:06,622 INFO [RS:1;f30636bb2128:33625 {}] regionserver.HRegionServer(1482): Serving as f30636bb2128,33625,1730980086216, RpcServer on f30636bb2128/172.17.0.2:33625, sessionid=0x1001a120c020002 2024-11-07T11:48:06,622 DEBUG [RS:1;f30636bb2128:33625 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-07T11:48:06,622 DEBUG [RS:1;f30636bb2128:33625 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f30636bb2128,33625,1730980086216 2024-11-07T11:48:06,623 DEBUG [RS:1;f30636bb2128:33625 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f30636bb2128,33625,1730980086216' 2024-11-07T11:48:06,623 DEBUG [RS:1;f30636bb2128:33625 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-07T11:48:06,623 INFO [RS:0;f30636bb2128:41943 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:06,623 INFO [RS:0;f30636bb2128:41943 {}] regionserver.HRegionServer(1482): Serving as f30636bb2128,41943,1730980086188, RpcServer on f30636bb2128/172.17.0.2:41943, sessionid=0x1001a120c020001 2024-11-07T11:48:06,623 DEBUG [RS:1;f30636bb2128:33625 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-07T11:48:06,624 DEBUG [RS:0;f30636bb2128:41943 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-07T11:48:06,624 DEBUG [RS:0;f30636bb2128:41943 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f30636bb2128,41943,1730980086188 2024-11-07T11:48:06,624 DEBUG [RS:0;f30636bb2128:41943 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f30636bb2128,41943,1730980086188' 2024-11-07T11:48:06,624 DEBUG [RS:0;f30636bb2128:41943 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-07T11:48:06,624 DEBUG [RS:1;f30636bb2128:33625 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-07T11:48:06,624 DEBUG [RS:1;f30636bb2128:33625 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-07T11:48:06,624 DEBUG [RS:1;f30636bb2128:33625 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f30636bb2128,33625,1730980086216 2024-11-07T11:48:06,624 DEBUG [RS:1;f30636bb2128:33625 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f30636bb2128,33625,1730980086216' 2024-11-07T11:48:06,624 DEBUG [RS:1;f30636bb2128:33625 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-07T11:48:06,624 DEBUG [RS:0;f30636bb2128:41943 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-07T11:48:06,625 DEBUG [RS:1;f30636bb2128:33625 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-07T11:48:06,625 DEBUG [RS:0;f30636bb2128:41943 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-07T11:48:06,625 DEBUG [RS:0;f30636bb2128:41943 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-07T11:48:06,625 DEBUG [RS:0;f30636bb2128:41943 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f30636bb2128,41943,1730980086188 2024-11-07T11:48:06,625 DEBUG [RS:0;f30636bb2128:41943 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f30636bb2128,41943,1730980086188' 2024-11-07T11:48:06,625 DEBUG [RS:0;f30636bb2128:41943 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-07T11:48:06,625 DEBUG [RS:1;f30636bb2128:33625 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-07T11:48:06,625 INFO [RS:1;f30636bb2128:33625 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-07T11:48:06,625 INFO [RS:1;f30636bb2128:33625 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-07T11:48:06,626 DEBUG [RS:0;f30636bb2128:41943 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-07T11:48:06,626 DEBUG [RS:0;f30636bb2128:41943 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-07T11:48:06,626 INFO [RS:0;f30636bb2128:41943 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-07T11:48:06,626 INFO [RS:0;f30636bb2128:41943 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-07T11:48:06,643 WARN [f30636bb2128:35521 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-07T11:48:06,726 INFO [RS:1;f30636bb2128:33625 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-07T11:48:06,727 INFO [RS:0;f30636bb2128:41943 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-07T11:48:06,727 INFO [RS:0;f30636bb2128:41943 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=f30636bb2128%2C41943%2C1730980086188, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/WALs/f30636bb2128,41943,1730980086188, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/oldWALs, maxLogs=55 2024-11-07T11:48:06,727 INFO [RS:1;f30636bb2128:33625 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=f30636bb2128%2C33625%2C1730980086216, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/WALs/f30636bb2128,33625,1730980086216, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/oldWALs, maxLogs=55 2024-11-07T11:48:06,747 INFO [RS:1;f30636bb2128:33625 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/WALs/f30636bb2128,33625,1730980086216/f30636bb2128%2C33625%2C1730980086216.1730980086730 2024-11-07T11:48:06,747 DEBUG [RS:1;f30636bb2128:33625 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-07T11:48:06,748 INFO [RS:0;f30636bb2128:41943 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/WALs/f30636bb2128,41943,1730980086188/f30636bb2128%2C41943%2C1730980086188.1730980086729 2024-11-07T11:48:06,748 DEBUG [RS:0;f30636bb2128:41943 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-07T11:48:06,894 DEBUG [f30636bb2128:35521 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-07T11:48:06,894 DEBUG [f30636bb2128:35521 {}] balancer.BalancerClusterState(204): Hosts are {f30636bb2128=0} racks are {/default-rack=0} 2024-11-07T11:48:06,895 DEBUG [f30636bb2128:35521 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-07T11:48:06,895 DEBUG [f30636bb2128:35521 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-07T11:48:06,895 DEBUG [f30636bb2128:35521 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-07T11:48:06,895 DEBUG [f30636bb2128:35521 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-07T11:48:06,895 INFO [f30636bb2128:35521 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-07T11:48:06,895 INFO [f30636bb2128:35521 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-07T11:48:06,895 DEBUG [f30636bb2128:35521 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-07T11:48:06,896 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=f30636bb2128,41943,1730980086188 2024-11-07T11:48:06,897 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f30636bb2128,41943,1730980086188, state=OPENING 2024-11-07T11:48:06,898 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-07T11:48:06,900 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33625-0x1001a120c020002, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:48:06,900 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35521-0x1001a120c020000, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:48:06,900 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41943-0x1001a120c020001, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:48:06,901 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T11:48:06,901 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:48:06,901 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=f30636bb2128,41943,1730980086188}] 2024-11-07T11:48:06,901 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:48:06,902 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:48:07,054 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-07T11:48:07,058 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45815, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-07T11:48:07,060 INFO [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-07T11:48:07,060 INFO [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T11:48:07,060 INFO [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-07T11:48:07,061 INFO [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=f30636bb2128%2C41943%2C1730980086188.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/WALs/f30636bb2128,41943,1730980086188, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/oldWALs, maxLogs=55 2024-11-07T11:48:07,080 INFO [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/WALs/f30636bb2128,41943,1730980086188/f30636bb2128%2C41943%2C1730980086188.meta.1730980087062.meta 2024-11-07T11:48:07,080 DEBUG [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-07T11:48:07,081 DEBUG [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-07T11:48:07,081 DEBUG [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-07T11:48:07,081 DEBUG [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-07T11:48:07,081 INFO [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-07T11:48:07,081 DEBUG [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-07T11:48:07,081 DEBUG [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:48:07,082 DEBUG [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-07T11:48:07,082 DEBUG [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-07T11:48:07,083 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-07T11:48:07,083 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-07T11:48:07,084 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:48:07,084 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T11:48:07,084 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-07T11:48:07,085 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-07T11:48:07,085 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:48:07,085 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T11:48:07,085 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-07T11:48:07,086 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-07T11:48:07,086 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:48:07,086 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T11:48:07,086 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-07T11:48:07,087 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-07T11:48:07,087 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:48:07,087 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T11:48:07,088 DEBUG [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-07T11:48:07,088 DEBUG [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740 2024-11-07T11:48:07,089 DEBUG [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740 2024-11-07T11:48:07,089 DEBUG [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-07T11:48:07,089 DEBUG [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-07T11:48:07,089 DEBUG [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-07T11:48:07,090 DEBUG [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-07T11:48:07,090 INFO [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71220880, jitterRate=0.061273813247680664}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-07T11:48:07,090 DEBUG [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-07T11:48:07,091 DEBUG [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1730980087082Writing region info on filesystem at 1730980087082Initializing all the Stores at 1730980087082Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730980087082Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730980087082Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730980087082Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730980087082Cleaning up temporary data from old regions at 1730980087089 (+7 ms)Running coprocessor post-open hooks at 1730980087090 (+1 ms)Region opened successfully at 1730980087091 (+1 ms) 2024-11-07T11:48:07,092 INFO [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1730980087054 2024-11-07T11:48:07,095 DEBUG [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-07T11:48:07,095 INFO [RS_OPEN_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-07T11:48:07,095 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=f30636bb2128,41943,1730980086188 2024-11-07T11:48:07,096 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f30636bb2128,41943,1730980086188, state=OPEN 2024-11-07T11:48:07,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35521-0x1001a120c020000, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:48:07,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33625-0x1001a120c020002, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:48:07,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41943-0x1001a120c020001, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T11:48:07,098 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:48:07,098 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:48:07,098 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T11:48:07,098 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=f30636bb2128,41943,1730980086188 2024-11-07T11:48:07,101 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-07T11:48:07,101 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f30636bb2128,41943,1730980086188 in 197 msec 2024-11-07T11:48:07,103 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-07T11:48:07,103 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 610 msec 2024-11-07T11:48:07,104 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-07T11:48:07,104 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-07T11:48:07,108 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T11:48:07,108 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f30636bb2128,41943,1730980086188, seqNum=-1] 2024-11-07T11:48:07,109 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T11:48:07,111 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42547, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T11:48:07,118 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 676 msec 2024-11-07T11:48:07,118 INFO [master/f30636bb2128:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1730980087118, completionTime=-1 2024-11-07T11:48:07,118 INFO [master/f30636bb2128:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-07T11:48:07,118 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-07T11:48:07,121 INFO [master/f30636bb2128:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-07T11:48:07,122 INFO [master/f30636bb2128:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1730980147121 2024-11-07T11:48:07,122 INFO [master/f30636bb2128:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1730980207122 2024-11-07T11:48:07,122 INFO [master/f30636bb2128:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec 2024-11-07T11:48:07,122 INFO [master/f30636bb2128:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f30636bb2128,35521,1730980086130-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:07,122 INFO [master/f30636bb2128:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f30636bb2128,35521,1730980086130-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:07,123 INFO [master/f30636bb2128:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f30636bb2128,35521,1730980086130-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:07,123 INFO [master/f30636bb2128:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-f30636bb2128:35521, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:07,123 INFO [master/f30636bb2128:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:07,124 INFO [master/f30636bb2128:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:07,126 DEBUG [master/f30636bb2128:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-07T11:48:07,130 INFO [master/f30636bb2128:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.875sec 2024-11-07T11:48:07,130 INFO [master/f30636bb2128:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-07T11:48:07,131 INFO [master/f30636bb2128:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-07T11:48:07,131 INFO [master/f30636bb2128:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-07T11:48:07,131 INFO [master/f30636bb2128:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-07T11:48:07,131 INFO [master/f30636bb2128:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-07T11:48:07,131 INFO [master/f30636bb2128:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f30636bb2128,35521,1730980086130-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-07T11:48:07,131 INFO [master/f30636bb2128:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f30636bb2128,35521,1730980086130-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-07T11:48:07,132 DEBUG [master/f30636bb2128:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-07T11:48:07,132 INFO [master/f30636bb2128:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-07T11:48:07,132 INFO [master/f30636bb2128:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f30636bb2128,35521,1730980086130-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:07,136 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@284befa8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T11:48:07,136 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f30636bb2128,35521,-1 for getting cluster id 2024-11-07T11:48:07,137 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T11:48:07,139 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2ec45d0e-1fd0-4ed6-b48a-056cb9ebaeee' 2024-11-07T11:48:07,140 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T11:48:07,140 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2ec45d0e-1fd0-4ed6-b48a-056cb9ebaeee" 2024-11-07T11:48:07,140 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54cb8d95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T11:48:07,140 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f30636bb2128,35521,-1] 2024-11-07T11:48:07,141 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T11:48:07,141 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T11:48:07,143 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59678, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T11:48:07,145 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2beb4aff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T11:48:07,145 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T11:48:07,147 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f30636bb2128,41943,1730980086188, seqNum=-1] 2024-11-07T11:48:07,147 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T11:48:07,149 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51266, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T11:48:07,151 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=f30636bb2128,35521,1730980086130 2024-11-07T11:48:07,152 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T11:48:07,153 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncConnectionImpl(321): The fetched master address is f30636bb2128,35521,1730980086130 2024-11-07T11:48:07,153 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@54cb293b 2024-11-07T11:48:07,153 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T11:48:07,155 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59680, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T11:48:07,156 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35521 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1730980087152', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T11:48:07,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35521 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1730980087152 2024-11-07T11:48:07,158 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1730980087152 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T11:48:07,159 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:48:07,159 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35521 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1730980087152" procId is: 4 2024-11-07T11:48:07,159 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1730980087152 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T11:48:07,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T11:48:07,176 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1730980087152-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0dcd4b66301e80bbdcdef0c188d6d908, NAME => 'testBlockEvictionOnGracefulStop1730980087152,,1730980087156.0dcd4b66301e80bbdcdef0c188d6d908.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1730980087152', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47 2024-11-07T11:48:07,182 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1730980087152-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1730980087152,,1730980087156.0dcd4b66301e80bbdcdef0c188d6d908.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:48:07,183 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1730980087152-pool-0 {}] regionserver.HRegion(1722): Closing 0dcd4b66301e80bbdcdef0c188d6d908, disabling compactions & flushes 2024-11-07T11:48:07,183 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1730980087152-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1730980087152,,1730980087156.0dcd4b66301e80bbdcdef0c188d6d908. 2024-11-07T11:48:07,183 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1730980087152-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1730980087152,,1730980087156.0dcd4b66301e80bbdcdef0c188d6d908. 2024-11-07T11:48:07,183 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1730980087152-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1730980087152,,1730980087156.0dcd4b66301e80bbdcdef0c188d6d908. after waiting 0 ms 2024-11-07T11:48:07,183 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1730980087152-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1730980087152,,1730980087156.0dcd4b66301e80bbdcdef0c188d6d908. 2024-11-07T11:48:07,183 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1730980087152-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1730980087152,,1730980087156.0dcd4b66301e80bbdcdef0c188d6d908. 2024-11-07T11:48:07,183 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1730980087152-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0dcd4b66301e80bbdcdef0c188d6d908: Waiting for close lock at 1730980087183Disabling compacts and flushes for region at 1730980087183Disabling writes for close at 1730980087183Writing region close event to WAL at 1730980087183Closed at 1730980087183 2024-11-07T11:48:07,184 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1730980087152 execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T11:48:07,184 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1730980087152,,1730980087156.0dcd4b66301e80bbdcdef0c188d6d908.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1730980087184"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730980087184"}]},"ts":"1730980087184"} 2024-11-07T11:48:07,187 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-07T11:48:07,188 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1730980087152 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T11:48:07,188 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1730980087152","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730980087188"}]},"ts":"1730980087188"} 2024-11-07T11:48:07,190 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1730980087152, state=ENABLING in hbase:meta 2024-11-07T11:48:07,190 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {f30636bb2128=0} racks are {/default-rack=0} 2024-11-07T11:48:07,191 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-07T11:48:07,191 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-07T11:48:07,191 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-07T11:48:07,191 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-07T11:48:07,191 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-07T11:48:07,191 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-07T11:48:07,191 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-07T11:48:07,191 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1730980087152, region=0dcd4b66301e80bbdcdef0c188d6d908, ASSIGN}] 2024-11-07T11:48:07,192 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1730980087152, region=0dcd4b66301e80bbdcdef0c188d6d908, ASSIGN 2024-11-07T11:48:07,193 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1730980087152, region=0dcd4b66301e80bbdcdef0c188d6d908, ASSIGN; state=OFFLINE, location=f30636bb2128,33625,1730980086216; forceNewPlan=false, retain=false 2024-11-07T11:48:07,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T11:48:07,344 INFO [f30636bb2128:35521 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-07T11:48:07,344 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=0dcd4b66301e80bbdcdef0c188d6d908, regionState=OPENING, regionLocation=f30636bb2128,33625,1730980086216 2024-11-07T11:48:07,347 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1730980087152, region=0dcd4b66301e80bbdcdef0c188d6d908, ASSIGN because future has completed 2024-11-07T11:48:07,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0dcd4b66301e80bbdcdef0c188d6d908, server=f30636bb2128,33625,1730980086216}] 2024-11-07T11:48:07,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T11:48:07,501 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-07T11:48:07,502 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40239, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-07T11:48:07,504 INFO [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1730980087152,,1730980087156.0dcd4b66301e80bbdcdef0c188d6d908. 2024-11-07T11:48:07,505 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 0dcd4b66301e80bbdcdef0c188d6d908, NAME => 'testBlockEvictionOnGracefulStop1730980087152,,1730980087156.0dcd4b66301e80bbdcdef0c188d6d908.', STARTKEY => '', ENDKEY => ''} 2024-11-07T11:48:07,505 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1730980087152 0dcd4b66301e80bbdcdef0c188d6d908 2024-11-07T11:48:07,505 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1730980087152,,1730980087156.0dcd4b66301e80bbdcdef0c188d6d908.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:48:07,505 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 0dcd4b66301e80bbdcdef0c188d6d908 2024-11-07T11:48:07,505 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 0dcd4b66301e80bbdcdef0c188d6d908 2024-11-07T11:48:07,506 INFO [StoreOpener-0dcd4b66301e80bbdcdef0c188d6d908-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 0dcd4b66301e80bbdcdef0c188d6d908 2024-11-07T11:48:07,510 INFO [StoreOpener-0dcd4b66301e80bbdcdef0c188d6d908-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0dcd4b66301e80bbdcdef0c188d6d908 columnFamilyName family 2024-11-07T11:48:07,510 DEBUG [StoreOpener-0dcd4b66301e80bbdcdef0c188d6d908-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:48:07,510 INFO [StoreOpener-0dcd4b66301e80bbdcdef0c188d6d908-1 {}] regionserver.HStore(327): Store=0dcd4b66301e80bbdcdef0c188d6d908/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T11:48:07,510 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 0dcd4b66301e80bbdcdef0c188d6d908 2024-11-07T11:48:07,511 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/default/testBlockEvictionOnGracefulStop1730980087152/0dcd4b66301e80bbdcdef0c188d6d908 2024-11-07T11:48:07,511 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/default/testBlockEvictionOnGracefulStop1730980087152/0dcd4b66301e80bbdcdef0c188d6d908 2024-11-07T11:48:07,511 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 0dcd4b66301e80bbdcdef0c188d6d908 2024-11-07T11:48:07,511 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 0dcd4b66301e80bbdcdef0c188d6d908 2024-11-07T11:48:07,512 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 0dcd4b66301e80bbdcdef0c188d6d908 2024-11-07T11:48:07,518 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/default/testBlockEvictionOnGracefulStop1730980087152/0dcd4b66301e80bbdcdef0c188d6d908/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T11:48:07,518 INFO [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 0dcd4b66301e80bbdcdef0c188d6d908; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74060041, jitterRate=0.10358060896396637}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T11:48:07,519 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0dcd4b66301e80bbdcdef0c188d6d908 2024-11-07T11:48:07,519 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 0dcd4b66301e80bbdcdef0c188d6d908: Running coprocessor pre-open hook at 1730980087505Writing region info on filesystem at 1730980087505Initializing all the Stores at 1730980087506 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730980087506Cleaning up temporary data from old regions at 1730980087511 (+5 ms)Running coprocessor post-open hooks at 1730980087519 (+8 ms)Region opened successfully at 1730980087519 2024-11-07T11:48:07,520 INFO [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1730980087152,,1730980087156.0dcd4b66301e80bbdcdef0c188d6d908., pid=6, masterSystemTime=1730980087500 2024-11-07T11:48:07,522 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1730980087152,,1730980087156.0dcd4b66301e80bbdcdef0c188d6d908. 2024-11-07T11:48:07,522 INFO [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1730980087152,,1730980087156.0dcd4b66301e80bbdcdef0c188d6d908. 2024-11-07T11:48:07,523 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=0dcd4b66301e80bbdcdef0c188d6d908, regionState=OPEN, openSeqNum=2, regionLocation=f30636bb2128,33625,1730980086216 2024-11-07T11:48:07,525 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0dcd4b66301e80bbdcdef0c188d6d908, server=f30636bb2128,33625,1730980086216 because future has completed 2024-11-07T11:48:07,528 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-07T11:48:07,528 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 0dcd4b66301e80bbdcdef0c188d6d908, server=f30636bb2128,33625,1730980086216 in 178 msec 2024-11-07T11:48:07,529 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-07T11:48:07,530 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1730980087152, region=0dcd4b66301e80bbdcdef0c188d6d908, ASSIGN in 337 msec 2024-11-07T11:48:07,530 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1730980087152 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T11:48:07,531 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1730980087152","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730980087531"}]},"ts":"1730980087531"} 2024-11-07T11:48:07,533 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1730980087152, state=ENABLED in hbase:meta 2024-11-07T11:48:07,534 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1730980087152 execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T11:48:07,535 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1730980087152 in 378 msec 2024-11-07T11:48:07,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T11:48:07,786 INFO [RPCClient-NioEventLoopGroup-5-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1730980087152 completed 2024-11-07T11:48:07,786 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1730980087152 get assigned. Timeout = 60000ms 2024-11-07T11:48:07,786 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T11:48:07,789 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1730980087152 assigned to meta. Checking AM states. 2024-11-07T11:48:07,790 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T11:48:07,790 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1730980087152 assigned. 2024-11-07T11:48:07,792 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1730980087152', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1730980087152,,1730980087156.0dcd4b66301e80bbdcdef0c188d6d908., hostname=f30636bb2128,33625,1730980086216, seqNum=2] 2024-11-07T11:48:07,793 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T11:48:07,795 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45248, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T11:48:07,798 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 0dcd4b66301e80bbdcdef0c188d6d908 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-07T11:48:07,808 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8f5ff55ee500451a9c507cbeb05bd8dd 2024-11-07T11:48:07,822 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/default/testBlockEvictionOnGracefulStop1730980087152/0dcd4b66301e80bbdcdef0c188d6d908/.tmp/family/8f5ff55ee500451a9c507cbeb05bd8dd is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-07T11:48:07,822 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/default/testBlockEvictionOnGracefulStop1730980087152/0dcd4b66301e80bbdcdef0c188d6d908/.tmp/family/8f5ff55ee500451a9c507cbeb05bd8dd 2024-11-07T11:48:07,825 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/default/testBlockEvictionOnGracefulStop1730980087152/0dcd4b66301e80bbdcdef0c188d6d908/.tmp/family/8f5ff55ee500451a9c507cbeb05bd8dd as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/default/testBlockEvictionOnGracefulStop1730980087152/0dcd4b66301e80bbdcdef0c188d6d908/family/8f5ff55ee500451a9c507cbeb05bd8dd 2024-11-07T11:48:07,826 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/default/testBlockEvictionOnGracefulStop1730980087152/0dcd4b66301e80bbdcdef0c188d6d908/family/8f5ff55ee500451a9c507cbeb05bd8dd, delay=919 ms 2024-11-07T11:48:07,827 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/default/testBlockEvictionOnGracefulStop1730980087152/0dcd4b66301e80bbdcdef0c188d6d908/family/8f5ff55ee500451a9c507cbeb05bd8dd, entries=2, sequenceid=6, filesize=4.9 K 2024-11-07T11:48:07,828 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 0dcd4b66301e80bbdcdef0c188d6d908 in 30ms, sequenceid=6, compaction requested=false 2024-11-07T11:48:07,828 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1730980087152' 2024-11-07T11:48:07,829 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 0dcd4b66301e80bbdcdef0c188d6d908: 2024-11-07T11:48:08,400 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T11:48:08,746 DEBUG [hfile-prefetch-1730980085582 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8f5ff55ee500451a9c507cbeb05bd8dd_0. 2024-11-07T11:48:08,747 DEBUG [hfile-prefetch-1730980085582 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 8f5ff55ee500451a9c507cbeb05bd8dd_117. 2024-11-07T11:48:08,747 DEBUG [hfile-prefetch-1730980085582 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/default/testBlockEvictionOnGracefulStop1730980087152/0dcd4b66301e80bbdcdef0c188d6d908/family/8f5ff55ee500451a9c507cbeb05bd8dd, with total blocks 2, and data blocks 1 2024-11-07T11:48:08,747 DEBUG [hfile-prefetch-1730980085582 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-07T11:48:08,747 DEBUG [hfile-prefetch-1730980085582 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/default/testBlockEvictionOnGracefulStop1730980087152/0dcd4b66301e80bbdcdef0c188d6d908/family/8f5ff55ee500451a9c507cbeb05bd8dd, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-07T11:48:08,747 DEBUG [hfile-prefetch-1730980085582 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/default/testBlockEvictionOnGracefulStop1730980087152/0dcd4b66301e80bbdcdef0c188d6d908/family/8f5ff55ee500451a9c507cbeb05bd8dd. Will sleep 100ms and try the verification again. 2024-11-07T11:48:08,798 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-07T11:48:08,799 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-07T11:48:08,799 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-07T11:48:08,799 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-07T11:48:08,802 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-07T11:48:08,802 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1730980088799 2024-11-07T11:48:08,802 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-07T11:48:08,829 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;f30636bb2128:33625,5,FailOnTimeoutGroup] 2024-11-07T11:48:08,829 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f30636bb2128,33625,1730980086216' ***** 2024-11-07T11:48:08,830 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-11-07T11:48:08,830 INFO [RS:1;f30636bb2128:33625 {}] bucket.BucketCache(1739): Disabling cache 2024-11-07T11:48:08,830 INFO [RS:1;f30636bb2128:33625 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5f86774-a615-bd53-b3f4-8c9383f47bad/bucket.persistence 2024-11-07T11:48:08,830 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-07T11:48:08,830 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-07T11:48:08,830 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-07T11:48:08,830 INFO [RS:1;f30636bb2128:33625 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-07T11:48:08,830 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-07T11:48:08,840 DEBUG [RS:1;f30636bb2128:33625 {}] bucket.BucketCache(1488): Thread RS:1;f30636bb2128:33625 started persisting bucket cache to file 2024-11-07T11:48:08,840 DEBUG [RS:1;f30636bb2128:33625 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-07T11:48:08,840 DEBUG [RS:1;f30636bb2128:33625 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-07T11:48:08,844 DEBUG [RS:1;f30636bb2128:33625 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-07T11:48:08,844 DEBUG [RS:1;f30636bb2128:33625 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1730980088840 2024-11-07T11:48:08,844 DEBUG [RS:1;f30636bb2128:33625 {}] bucket.BucketCache(1510): Thread RS:1;f30636bb2128:33625 finished persisting bucket cache to file, renaming 2024-11-07T11:48:08,844 INFO [RS:1;f30636bb2128:33625 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-07T11:48:08,844 INFO [RS:1;f30636bb2128:33625 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-07T11:48:08,844 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-07T11:48:08,844 INFO [RS:1;f30636bb2128:33625 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-07T11:48:08,844 INFO [RS:1;f30636bb2128:33625 {}] regionserver.HRegionServer(3091): Received CLOSE for 0dcd4b66301e80bbdcdef0c188d6d908 2024-11-07T11:48:08,845 INFO [RS:1;f30636bb2128:33625 {}] regionserver.HRegionServer(959): stopping server f30636bb2128,33625,1730980086216 2024-11-07T11:48:08,845 INFO [RS:1;f30636bb2128:33625 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-07T11:48:08,845 INFO [RS:1;f30636bb2128:33625 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;f30636bb2128:33625. 2024-11-07T11:48:08,845 INFO [RS_CLOSE_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1730980087152,,1730980087156.0dcd4b66301e80bbdcdef0c188d6d908. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-07T11:48:08,845 DEBUG [RS:1;f30636bb2128:33625 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T11:48:08,845 DEBUG [RS_CLOSE_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0dcd4b66301e80bbdcdef0c188d6d908, disabling compactions & flushes 2024-11-07T11:48:08,845 DEBUG [RS:1;f30636bb2128:33625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T11:48:08,845 INFO [RS_CLOSE_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1730980087152,,1730980087156.0dcd4b66301e80bbdcdef0c188d6d908. 2024-11-07T11:48:08,845 DEBUG [RS_CLOSE_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1730980087152,,1730980087156.0dcd4b66301e80bbdcdef0c188d6d908. 2024-11-07T11:48:08,845 INFO [RS:1;f30636bb2128:33625 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-07T11:48:08,845 DEBUG [RS_CLOSE_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1730980087152,,1730980087156.0dcd4b66301e80bbdcdef0c188d6d908. after waiting 0 ms 2024-11-07T11:48:08,845 DEBUG [RS:1;f30636bb2128:33625 {}] regionserver.HRegionServer(1325): Online Regions={0dcd4b66301e80bbdcdef0c188d6d908=testBlockEvictionOnGracefulStop1730980087152,,1730980087156.0dcd4b66301e80bbdcdef0c188d6d908.} 2024-11-07T11:48:08,845 DEBUG [RS_CLOSE_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1730980087152,,1730980087156.0dcd4b66301e80bbdcdef0c188d6d908. 2024-11-07T11:48:08,845 DEBUG [RS:1;f30636bb2128:33625 {}] regionserver.HRegionServer(1351): Waiting on 0dcd4b66301e80bbdcdef0c188d6d908 2024-11-07T11:48:08,846 DEBUG [StoreFileCloser-0dcd4b66301e80bbdcdef0c188d6d908-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/default/testBlockEvictionOnGracefulStop1730980087152/0dcd4b66301e80bbdcdef0c188d6d908/family/8f5ff55ee500451a9c507cbeb05bd8dd 2024-11-07T11:48:08,846 WARN [hfile-prefetch-1730980085582 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/default/testBlockEvictionOnGracefulStop1730980087152/0dcd4b66301e80bbdcdef0c188d6d908/family/8f5ff55ee500451a9c507cbeb05bd8dd, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2338) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2321) ~[classes/:?] ... 9 more 2024-11-07T11:48:08,846 DEBUG [hfile-prefetch-1730980085582 {}] hfile.PrefetchExecutor(121): Prefetch completed for 8f5ff55ee500451a9c507cbeb05bd8dd 2024-11-07T11:48:08,850 DEBUG [RS_CLOSE_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/default/testBlockEvictionOnGracefulStop1730980087152/0dcd4b66301e80bbdcdef0c188d6d908/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-07T11:48:08,851 INFO [RS_CLOSE_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1730980087152,,1730980087156.0dcd4b66301e80bbdcdef0c188d6d908. 2024-11-07T11:48:08,851 DEBUG [RS_CLOSE_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0dcd4b66301e80bbdcdef0c188d6d908: Waiting for close lock at 1730980088845Running coprocessor pre-close hooks at 1730980088845Disabling compacts and flushes for region at 1730980088845Disabling writes for close at 1730980088845Writing region close event to WAL at 1730980088846 (+1 ms)Running coprocessor post-close hooks at 1730980088850 (+4 ms)Closed at 1730980088850 2024-11-07T11:48:08,851 DEBUG [RS_CLOSE_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1730980087152,,1730980087156.0dcd4b66301e80bbdcdef0c188d6d908. 2024-11-07T11:48:08,893 INFO [regionserver/f30636bb2128:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T11:48:09,045 INFO [RS:1;f30636bb2128:33625 {}] regionserver.HRegionServer(976): stopping server f30636bb2128,33625,1730980086216; all regions closed. 2024-11-07T11:48:09,047 DEBUG [RS:1;f30636bb2128:33625 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/oldWALs 2024-11-07T11:48:09,048 INFO [RS:1;f30636bb2128:33625 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f30636bb2128%2C33625%2C1730980086216:(num 1730980086730) 2024-11-07T11:48:09,048 DEBUG [RS:1;f30636bb2128:33625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T11:48:09,048 INFO [RS:1;f30636bb2128:33625 {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T11:48:09,048 INFO [RS:1;f30636bb2128:33625 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-07T11:48:09,048 INFO [RS:1;f30636bb2128:33625 {}] hbase.ChoreService(370): Chore service for: regionserver/f30636bb2128:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-07T11:48:09,048 INFO [RS:1;f30636bb2128:33625 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-07T11:48:09,048 INFO [RS:1;f30636bb2128:33625 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-07T11:48:09,048 INFO [RS:1;f30636bb2128:33625 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-07T11:48:09,048 INFO [regionserver/f30636bb2128:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-07T11:48:09,048 INFO [RS:1;f30636bb2128:33625 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-07T11:48:09,048 INFO [RS:1;f30636bb2128:33625 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33625 2024-11-07T11:48:09,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33625-0x1001a120c020002, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f30636bb2128,33625,1730980086216 2024-11-07T11:48:09,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35521-0x1001a120c020000, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-07T11:48:09,053 INFO [RS:1;f30636bb2128:33625 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-07T11:48:09,054 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f30636bb2128,33625,1730980086216] 2024-11-07T11:48:09,056 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f30636bb2128,33625,1730980086216 already deleted, retry=false 2024-11-07T11:48:09,056 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of f30636bb2128,33625,1730980086216 on f30636bb2128,35521,1730980086130 2024-11-07T11:48:09,061 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure f30636bb2128,33625,1730980086216, splitWal=true, meta=false 2024-11-07T11:48:09,063 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for f30636bb2128,33625,1730980086216 (carryingMeta=false) f30636bb2128,33625,1730980086216/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@29af2584[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-07T11:48:09,063 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure f30636bb2128,33625,1730980086216, splitWal=true, meta=false 2024-11-07T11:48:09,064 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): f30636bb2128,33625,1730980086216 had 1 regions 2024-11-07T11:48:09,065 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure f30636bb2128,33625,1730980086216, splitWal=true, meta=false, isMeta: false 2024-11-07T11:48:09,065 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/WALs/f30636bb2128,33625,1730980086216-splitting 2024-11-07T11:48:09,066 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/WALs/f30636bb2128,33625,1730980086216-splitting dir is empty, no logs to split. 2024-11-07T11:48:09,066 INFO [PEWorker-2 {}] master.SplitWALManager(105): f30636bb2128,33625,1730980086216 WAL count=0, meta=false 2024-11-07T11:48:09,067 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/WALs/f30636bb2128,33625,1730980086216-splitting dir is empty, no logs to split. 2024-11-07T11:48:09,067 INFO [PEWorker-2 {}] master.SplitWALManager(105): f30636bb2128,33625,1730980086216 WAL count=0, meta=false 2024-11-07T11:48:09,067 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if f30636bb2128,33625,1730980086216 WAL splitting is done? wals=0, meta=false 2024-11-07T11:48:09,068 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for f30636bb2128,33625,1730980086216 failed, ignore...File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/WALs/f30636bb2128,33625,1730980086216-splitting does not exist 2024-11-07T11:48:09,069 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1730980087152, region=0dcd4b66301e80bbdcdef0c188d6d908, ASSIGN}] 2024-11-07T11:48:09,070 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1730980087152, region=0dcd4b66301e80bbdcdef0c188d6d908, ASSIGN 2024-11-07T11:48:09,071 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1730980087152, region=0dcd4b66301e80bbdcdef0c188d6d908, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-07T11:48:09,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33625-0x1001a120c020002, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T11:48:09,155 INFO [RS:1;f30636bb2128:33625 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-07T11:48:09,155 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33625-0x1001a120c020002, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T11:48:09,155 INFO [RS:1;f30636bb2128:33625 {}] regionserver.HRegionServer(1031): Exiting; stopping=f30636bb2128,33625,1730980086216; zookeeper connection closed. 2024-11-07T11:48:09,155 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1fef876c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1fef876c 2024-11-07T11:48:09,221 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=0dcd4b66301e80bbdcdef0c188d6d908, regionState=OPENING, regionLocation=f30636bb2128,41943,1730980086188 2024-11-07T11:48:09,224 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1730980087152, region=0dcd4b66301e80bbdcdef0c188d6d908, ASSIGN because future has completed 2024-11-07T11:48:09,225 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0dcd4b66301e80bbdcdef0c188d6d908, server=f30636bb2128,41943,1730980086188}] 2024-11-07T11:48:09,345 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f30636bb2128:0 server-side Connection retries=45 2024-11-07T11:48:09,346 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T11:48:09,346 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-07T11:48:09,346 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-07T11:48:09,346 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T11:48:09,346 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-07T11:48:09,346 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-07T11:48:09,346 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-07T11:48:09,347 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39281 2024-11-07T11:48:09,348 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39281 connecting to ZooKeeper ensemble=127.0.0.1:63417 2024-11-07T11:48:09,348 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T11:48:09,348 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T11:48:09,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:392810x0, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-07T11:48:09,352 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39281-0x1001a120c020003 connected 2024-11-07T11:48:09,352 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:39281-0x1001a120c020003, quorum=127.0.0.1:63417, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-07T11:48:09,352 DEBUG [pool-297-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-07T11:48:09,353 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-07T11:48:09,353 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5f86774-a615-bd53-b3f4-8c9383f47bad/bucket.cache 2024-11-07T11:48:09,353 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-07T11:48:09,353 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-07T11:48:09,353 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-07T11:48:09,355 INFO [Thread-202 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-07T11:48:09,356 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5f86774-a615-bd53-b3f4-8c9383f47bad/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5f86774-a615-bd53-b3f4-8c9383f47bad/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-07T11:48:09,356 INFO [Thread-202 {}] bucket.BucketCache(1553): Reading new chunked format of persistence. 2024-11-07T11:48:09,356 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-07T11:48:09,357 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:39281-0x1001a120c020003, quorum=127.0.0.1:63417, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-07T11:48:09,357 INFO [Thread-202 {}] bucket.BucketCache(1706): Retrieved 1 of chunks with blockCount = 1. 2024-11-07T11:48:09,359 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39281-0x1001a120c020003, quorum=127.0.0.1:63417, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-07T11:48:09,359 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39281 2024-11-07T11:48:09,359 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39281 2024-11-07T11:48:09,359 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39281 2024-11-07T11:48:09,360 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39281 2024-11-07T11:48:09,360 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39281 2024-11-07T11:48:09,361 DEBUG [Thread-202 {}] bucket.BucketCache(1570): Updating region size map after retrieving cached file list 2024-11-07T11:48:09,361 INFO [Thread-202 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-07T11:48:09,361 INFO [Thread-202 {}] bucket.BucketCache(1563): Bucket cache retrieved from file successfully with size: 1 2024-11-07T11:48:09,361 INFO [Thread-202 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5f86774-a615-bd53-b3f4-8c9383f47bad/bucket.persistence is complete. 2024-11-07T11:48:09,361 INFO [RS:2;f30636bb2128:39281 {}] regionserver.HRegionServer(746): ClusterId : 2ec45d0e-1fd0-4ed6-b48a-056cb9ebaeee 2024-11-07T11:48:09,362 DEBUG [RS:2;f30636bb2128:39281 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-07T11:48:09,364 DEBUG [RS:2;f30636bb2128:39281 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-07T11:48:09,364 DEBUG [RS:2;f30636bb2128:39281 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-07T11:48:09,366 DEBUG [RS:2;f30636bb2128:39281 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-07T11:48:09,367 DEBUG [RS:2;f30636bb2128:39281 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1583fec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f30636bb2128/172.17.0.2:0 2024-11-07T11:48:09,378 DEBUG [RS:2;f30636bb2128:39281 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;f30636bb2128:39281 2024-11-07T11:48:09,378 INFO [RS:2;f30636bb2128:39281 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-07T11:48:09,378 INFO [RS:2;f30636bb2128:39281 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-07T11:48:09,378 DEBUG [RS:2;f30636bb2128:39281 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-07T11:48:09,378 INFO [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1730980087152,,1730980087156.0dcd4b66301e80bbdcdef0c188d6d908. 2024-11-07T11:48:09,379 INFO [RS:2;f30636bb2128:39281 {}] regionserver.HRegionServer(2659): reportForDuty to master=f30636bb2128,35521,1730980086130 with port=39281, startcode=1730980089345 2024-11-07T11:48:09,379 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 0dcd4b66301e80bbdcdef0c188d6d908, NAME => 'testBlockEvictionOnGracefulStop1730980087152,,1730980087156.0dcd4b66301e80bbdcdef0c188d6d908.', STARTKEY => '', ENDKEY => ''} 2024-11-07T11:48:09,379 DEBUG [RS:2;f30636bb2128:39281 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-07T11:48:09,379 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1730980087152 0dcd4b66301e80bbdcdef0c188d6d908 2024-11-07T11:48:09,379 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1730980087152,,1730980087156.0dcd4b66301e80bbdcdef0c188d6d908.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T11:48:09,380 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 0dcd4b66301e80bbdcdef0c188d6d908 2024-11-07T11:48:09,380 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 0dcd4b66301e80bbdcdef0c188d6d908 2024-11-07T11:48:09,381 INFO [StoreOpener-0dcd4b66301e80bbdcdef0c188d6d908-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 0dcd4b66301e80bbdcdef0c188d6d908 2024-11-07T11:48:09,381 INFO [StoreOpener-0dcd4b66301e80bbdcdef0c188d6d908-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0dcd4b66301e80bbdcdef0c188d6d908 columnFamilyName family 2024-11-07T11:48:09,381 DEBUG [StoreOpener-0dcd4b66301e80bbdcdef0c188d6d908-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T11:48:09,382 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55109, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-07T11:48:09,382 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35521 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f30636bb2128,39281,1730980089345 2024-11-07T11:48:09,382 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35521 {}] master.ServerManager(517): Registering regionserver=f30636bb2128,39281,1730980089345 2024-11-07T11:48:09,383 DEBUG [StoreFileOpener-0dcd4b66301e80bbdcdef0c188d6d908-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/default/testBlockEvictionOnGracefulStop1730980087152/0dcd4b66301e80bbdcdef0c188d6d908/family/8f5ff55ee500451a9c507cbeb05bd8dd, delay=918 ms 2024-11-07T11:48:09,384 DEBUG [RS:2;f30636bb2128:39281 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47 2024-11-07T11:48:09,384 DEBUG [RS:2;f30636bb2128:39281 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-07T11:48:09,384 DEBUG [RS:2;f30636bb2128:39281 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-07T11:48:09,385 DEBUG [StoreOpener-0dcd4b66301e80bbdcdef0c188d6d908-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/default/testBlockEvictionOnGracefulStop1730980087152/0dcd4b66301e80bbdcdef0c188d6d908/family/8f5ff55ee500451a9c507cbeb05bd8dd 2024-11-07T11:48:09,385 INFO [StoreOpener-0dcd4b66301e80bbdcdef0c188d6d908-1 {}] regionserver.HStore(327): Store=0dcd4b66301e80bbdcdef0c188d6d908/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T11:48:09,385 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 0dcd4b66301e80bbdcdef0c188d6d908 2024-11-07T11:48:09,386 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35521-0x1001a120c020000, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-07T11:48:09,386 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/default/testBlockEvictionOnGracefulStop1730980087152/0dcd4b66301e80bbdcdef0c188d6d908 2024-11-07T11:48:09,386 DEBUG [RS:2;f30636bb2128:39281 {}] zookeeper.ZKUtil(111): regionserver:39281-0x1001a120c020003, quorum=127.0.0.1:63417, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f30636bb2128,39281,1730980089345 2024-11-07T11:48:09,386 WARN [RS:2;f30636bb2128:39281 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-07T11:48:09,386 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f30636bb2128,39281,1730980089345] 2024-11-07T11:48:09,386 INFO [RS:2;f30636bb2128:39281 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T11:48:09,387 DEBUG [RS:2;f30636bb2128:39281 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/WALs/f30636bb2128,39281,1730980089345 2024-11-07T11:48:09,387 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/default/testBlockEvictionOnGracefulStop1730980087152/0dcd4b66301e80bbdcdef0c188d6d908 2024-11-07T11:48:09,387 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 0dcd4b66301e80bbdcdef0c188d6d908 2024-11-07T11:48:09,387 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 0dcd4b66301e80bbdcdef0c188d6d908 2024-11-07T11:48:09,389 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 0dcd4b66301e80bbdcdef0c188d6d908 2024-11-07T11:48:09,389 INFO [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 0dcd4b66301e80bbdcdef0c188d6d908; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63394568, jitterRate=-0.055347323417663574}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-07T11:48:09,389 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0dcd4b66301e80bbdcdef0c188d6d908 2024-11-07T11:48:09,390 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 0dcd4b66301e80bbdcdef0c188d6d908: Running coprocessor pre-open hook at 1730980089380Writing region info on filesystem at 1730980089380Initializing all the Stores at 1730980089380Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730980089380Cleaning up temporary data from old regions at 1730980089387 (+7 ms)Running coprocessor post-open hooks at 1730980089390 (+3 ms)Region opened successfully at 1730980089390 2024-11-07T11:48:09,391 INFO [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1730980087152,,1730980087156.0dcd4b66301e80bbdcdef0c188d6d908., pid=9, masterSystemTime=1730980089376 2024-11-07T11:48:09,391 INFO [RS:2;f30636bb2128:39281 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-07T11:48:09,394 DEBUG [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1730980087152,,1730980087156.0dcd4b66301e80bbdcdef0c188d6d908. 2024-11-07T11:48:09,394 INFO [RS_OPEN_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1730980087152,,1730980087156.0dcd4b66301e80bbdcdef0c188d6d908. 2024-11-07T11:48:09,394 INFO [RS:2;f30636bb2128:39281 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-07T11:48:09,394 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=0dcd4b66301e80bbdcdef0c188d6d908, regionState=OPEN, openSeqNum=10, regionLocation=f30636bb2128,41943,1730980086188 2024-11-07T11:48:09,394 INFO [RS:2;f30636bb2128:39281 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-07T11:48:09,395 INFO [RS:2;f30636bb2128:39281 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:09,395 INFO [RS:2;f30636bb2128:39281 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-07T11:48:09,395 INFO [RS:2;f30636bb2128:39281 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-07T11:48:09,395 INFO [RS:2;f30636bb2128:39281 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:09,396 DEBUG [RS:2;f30636bb2128:39281 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:09,396 DEBUG [RS:2;f30636bb2128:39281 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:09,396 DEBUG [RS:2;f30636bb2128:39281 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:09,396 DEBUG [RS:2;f30636bb2128:39281 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:09,396 DEBUG [RS:2;f30636bb2128:39281 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:09,396 DEBUG [RS:2;f30636bb2128:39281 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f30636bb2128:0, corePoolSize=2, maxPoolSize=2 2024-11-07T11:48:09,396 DEBUG [RS:2;f30636bb2128:39281 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:09,396 DEBUG [RS:2;f30636bb2128:39281 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:09,396 DEBUG [RS:2;f30636bb2128:39281 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:09,396 DEBUG [RS:2;f30636bb2128:39281 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:09,396 DEBUG [RS:2;f30636bb2128:39281 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:09,396 DEBUG [RS:2;f30636bb2128:39281 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f30636bb2128:0, corePoolSize=1, maxPoolSize=1 2024-11-07T11:48:09,396 DEBUG [RS:2;f30636bb2128:39281 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f30636bb2128:0, corePoolSize=3, maxPoolSize=3 2024-11-07T11:48:09,397 DEBUG [RS:2;f30636bb2128:39281 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f30636bb2128:0, corePoolSize=3, maxPoolSize=3 2024-11-07T11:48:09,397 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0dcd4b66301e80bbdcdef0c188d6d908, server=f30636bb2128,41943,1730980086188 because future has completed 2024-11-07T11:48:09,400 INFO [RS:2;f30636bb2128:39281 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:09,401 INFO [RS:2;f30636bb2128:39281 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:09,401 INFO [RS:2;f30636bb2128:39281 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:09,401 INFO [RS:2;f30636bb2128:39281 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:09,401 INFO [RS:2;f30636bb2128:39281 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:09,401 INFO [RS:2;f30636bb2128:39281 {}] hbase.ChoreService(168): Chore ScheduledChore name=f30636bb2128,39281,1730980089345-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-07T11:48:09,403 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-07T11:48:09,403 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 0dcd4b66301e80bbdcdef0c188d6d908, server=f30636bb2128,41943,1730980086188 in 177 msec 2024-11-07T11:48:09,404 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-07T11:48:09,405 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1730980087152, region=0dcd4b66301e80bbdcdef0c188d6d908, ASSIGN in 334 msec 2024-11-07T11:48:09,405 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server f30636bb2128,33625,1730980086216 after splitting done 2024-11-07T11:48:09,406 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure f30636bb2128,33625,1730980086216, splitWal=true, meta=false in 348 msec 2024-11-07T11:48:09,419 INFO [RS:2;f30636bb2128:39281 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-07T11:48:09,420 INFO [RS:2;f30636bb2128:39281 {}] hbase.ChoreService(168): Chore ScheduledChore name=f30636bb2128,39281,1730980089345-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:09,420 INFO [RS:2;f30636bb2128:39281 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:09,420 INFO [RS:2;f30636bb2128:39281 {}] regionserver.Replication(171): f30636bb2128,39281,1730980089345 started 2024-11-07T11:48:09,435 INFO [RS:2;f30636bb2128:39281 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T11:48:09,435 INFO [RS:2;f30636bb2128:39281 {}] regionserver.HRegionServer(1482): Serving as f30636bb2128,39281,1730980089345, RpcServer on f30636bb2128/172.17.0.2:39281, sessionid=0x1001a120c020003 2024-11-07T11:48:09,435 DEBUG [RS:2;f30636bb2128:39281 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-07T11:48:09,435 DEBUG [RS:2;f30636bb2128:39281 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f30636bb2128,39281,1730980089345 2024-11-07T11:48:09,435 DEBUG [RS:2;f30636bb2128:39281 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f30636bb2128,39281,1730980089345' 2024-11-07T11:48:09,435 DEBUG [RS:2;f30636bb2128:39281 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-07T11:48:09,436 DEBUG [RS:2;f30636bb2128:39281 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-07T11:48:09,436 DEBUG [RS:2;f30636bb2128:39281 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-07T11:48:09,436 DEBUG [RS:2;f30636bb2128:39281 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-07T11:48:09,436 DEBUG [RS:2;f30636bb2128:39281 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f30636bb2128,39281,1730980089345 2024-11-07T11:48:09,436 DEBUG [RS:2;f30636bb2128:39281 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f30636bb2128,39281,1730980089345' 2024-11-07T11:48:09,436 DEBUG [RS:2;f30636bb2128:39281 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-07T11:48:09,437 DEBUG [RS:2;f30636bb2128:39281 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-07T11:48:09,437 DEBUG [RS:2;f30636bb2128:39281 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-07T11:48:09,437 INFO [RS:2;f30636bb2128:39281 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-07T11:48:09,437 INFO [RS:2;f30636bb2128:39281 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-07T11:48:09,538 INFO [RS:2;f30636bb2128:39281 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-07T11:48:09,538 INFO [RS:2;f30636bb2128:39281 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=f30636bb2128%2C39281%2C1730980089345, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/WALs/f30636bb2128,39281,1730980089345, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/oldWALs, maxLogs=55 2024-11-07T11:48:09,557 INFO [RS:2;f30636bb2128:39281 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/WALs/f30636bb2128,39281,1730980089345/f30636bb2128%2C39281%2C1730980089345.1730980089539 2024-11-07T11:48:09,557 DEBUG [RS:2;f30636bb2128:39281 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-07T11:48:09,935 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-07T11:48:09,935 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T11:48:09,935 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T11:48:09,935 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T11:48:09,936 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T11:48:09,936 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T11:48:09,936 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-07T11:48:09,936 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1838120439, stopped=false 2024-11-07T11:48:09,936 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=f30636bb2128,35521,1730980086130 2024-11-07T11:48:09,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41943-0x1001a120c020001, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-07T11:48:09,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39281-0x1001a120c020003, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-07T11:48:09,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35521-0x1001a120c020000, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-07T11:48:09,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41943-0x1001a120c020001, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:48:09,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39281-0x1001a120c020003, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:48:09,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35521-0x1001a120c020000, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:48:09,938 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T11:48:09,939 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T11:48:09,939 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T11:48:09,939 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T11:48:09,939 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f30636bb2128,41943,1730980086188' ***** 2024-11-07T11:48:09,939 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41943-0x1001a120c020001, quorum=127.0.0.1:63417, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T11:48:09,939 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-07T11:48:09,939 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f30636bb2128,39281,1730980089345' ***** 2024-11-07T11:48:09,939 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-07T11:48:09,940 INFO [RS:0;f30636bb2128:41943 {}] bucket.BucketCache(1739): Disabling cache 2024-11-07T11:48:09,940 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35521-0x1001a120c020000, quorum=127.0.0.1:63417, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T11:48:09,940 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39281-0x1001a120c020003, quorum=127.0.0.1:63417, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T11:48:09,940 INFO [RS:0;f30636bb2128:41943 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5f86774-a615-bd53-b3f4-8c9383f47bad/bucket.persistence 2024-11-07T11:48:09,940 INFO [RS:2;f30636bb2128:39281 {}] bucket.BucketCache(1739): Disabling cache 2024-11-07T11:48:09,940 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-07T11:48:09,940 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-07T11:48:09,940 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-07T11:48:09,941 INFO [RS:2;f30636bb2128:39281 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5f86774-a615-bd53-b3f4-8c9383f47bad/bucket.persistence 2024-11-07T11:48:09,941 INFO [RS:0;f30636bb2128:41943 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-07T11:48:09,941 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-07T11:48:09,941 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-07T11:48:09,941 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-07T11:48:09,941 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-07T11:48:09,941 INFO [RS:2;f30636bb2128:39281 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-07T11:48:09,941 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-07T11:48:09,951 DEBUG [RS:0;f30636bb2128:41943 {}] bucket.BucketCache(1488): Thread RS:0;f30636bb2128:41943 started persisting bucket cache to file 2024-11-07T11:48:09,951 DEBUG [RS:0;f30636bb2128:41943 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-07T11:48:09,951 DEBUG [RS:0;f30636bb2128:41943 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-07T11:48:09,951 DEBUG [RS:2;f30636bb2128:39281 {}] bucket.BucketCache(1488): Thread RS:2;f30636bb2128:39281 started persisting bucket cache to file 2024-11-07T11:48:09,952 DEBUG [RS:2;f30636bb2128:39281 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-07T11:48:09,952 DEBUG [RS:2;f30636bb2128:39281 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-07T11:48:09,955 DEBUG [RS:0;f30636bb2128:41943 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-07T11:48:09,955 DEBUG [RS:0;f30636bb2128:41943 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1730980089951 2024-11-07T11:48:09,955 DEBUG [RS:0;f30636bb2128:41943 {}] bucket.BucketCache(1510): Thread RS:0;f30636bb2128:41943 finished persisting bucket cache to file, renaming 2024-11-07T11:48:09,955 WARN [Thread-210 {}] util.Shell$1(1036): Error reading the error stream java.io.IOException: Stream closed at java.io.BufferedInputStream.getBufIfOpen(BufferedInputStream.java:168) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:334) ~[?:?] at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:281) ~[?:?] at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:324) ~[?:?] at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:189) ~[?:?] at java.io.InputStreamReader.read(InputStreamReader.java:177) ~[?:?] at java.io.BufferedReader.fill(BufferedReader.java:162) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:329) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:396) ~[?:?] at org.apache.hadoop.util.Shell$1.run(Shell.java:1025) ~[hadoop-common-3.4.1.jar:?] 2024-11-07T11:48:09,955 ERROR [RS:2;f30636bb2128:39281 {}] bucket.PersistentIOEngine(78): Calculating checksum failed, because of java.io.IOException: Stream closed at java.io.BufferedInputStream.getBufIfOpen(BufferedInputStream.java:168) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:334) ~[?:?] at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:281) ~[?:?] at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:324) ~[?:?] at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:189) ~[?:?] at java.io.InputStreamReader.read(InputStreamReader.java:177) ~[?:?] at java.io.BufferedReader.fill(BufferedReader.java:162) ~[?:?] at java.io.BufferedReader.read1(BufferedReader.java:213) ~[?:?] at java.io.BufferedReader.read(BufferedReader.java:287) ~[?:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.parseExecResult(Shell.java:1295) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.runCommand(Shell.java:1054) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.getFileSize(PersistentIOEngine.java:94) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.calculateChecksum(PersistentIOEngine.java:71) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.toPB(BucketProtoUtils.java:61) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.serializeAsPB(BucketProtoUtils.java:75) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistChunkedBackingMap(BucketCache.java:1679) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1497) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.shutdown(BucketCache.java:1775) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.shutdown(CombinedBlockCache.java:169) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:927) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T11:48:09,955 DEBUG [RS:2;f30636bb2128:39281 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-07T11:48:09,955 DEBUG [RS:2;f30636bb2128:39281 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1730980089951 2024-11-07T11:48:09,955 DEBUG [RS:2;f30636bb2128:39281 {}] bucket.BucketCache(1510): Thread RS:2;f30636bb2128:39281 finished persisting bucket cache to file, renaming 2024-11-07T11:48:09,956 WARN [RS:2;f30636bb2128:39281 {}] bucket.BucketCache(1513): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-07T11:48:09,956 INFO [RS:0;f30636bb2128:41943 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-07T11:48:09,956 INFO [RS:2;f30636bb2128:39281 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-07T11:48:09,956 INFO [RS:0;f30636bb2128:41943 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-07T11:48:09,956 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-07T11:48:09,956 INFO [RS:0;f30636bb2128:41943 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-07T11:48:09,956 INFO [RS:2;f30636bb2128:39281 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-07T11:48:09,956 INFO [RS:0;f30636bb2128:41943 {}] regionserver.HRegionServer(3091): Received CLOSE for 0dcd4b66301e80bbdcdef0c188d6d908 2024-11-07T11:48:09,956 INFO [RS:2;f30636bb2128:39281 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-07T11:48:09,956 INFO [RS:2;f30636bb2128:39281 {}] regionserver.HRegionServer(959): stopping server f30636bb2128,39281,1730980089345 2024-11-07T11:48:09,956 INFO [RS:2;f30636bb2128:39281 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-07T11:48:09,956 INFO [RS:2;f30636bb2128:39281 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;f30636bb2128:39281. 2024-11-07T11:48:09,956 INFO [RS:0;f30636bb2128:41943 {}] regionserver.HRegionServer(959): stopping server f30636bb2128,41943,1730980086188 2024-11-07T11:48:09,956 INFO [RS:0;f30636bb2128:41943 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-07T11:48:09,956 DEBUG [RS:2;f30636bb2128:39281 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T11:48:09,956 INFO [RS:0;f30636bb2128:41943 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;f30636bb2128:41943. 2024-11-07T11:48:09,956 DEBUG [RS:2;f30636bb2128:39281 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T11:48:09,956 DEBUG [RS:0;f30636bb2128:41943 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T11:48:09,956 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-07T11:48:09,957 DEBUG [RS:0;f30636bb2128:41943 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T11:48:09,956 INFO [RS_CLOSE_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1730980087152,,1730980087156.0dcd4b66301e80bbdcdef0c188d6d908. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-07T11:48:09,957 INFO [RS:2;f30636bb2128:39281 {}] regionserver.HRegionServer(976): stopping server f30636bb2128,39281,1730980089345; all regions closed. 2024-11-07T11:48:09,957 DEBUG [RS_CLOSE_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0dcd4b66301e80bbdcdef0c188d6d908, disabling compactions & flushes 2024-11-07T11:48:09,957 INFO [RS_CLOSE_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1730980087152,,1730980087156.0dcd4b66301e80bbdcdef0c188d6d908. 2024-11-07T11:48:09,957 DEBUG [RS_CLOSE_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1730980087152,,1730980087156.0dcd4b66301e80bbdcdef0c188d6d908. 2024-11-07T11:48:09,957 INFO [RS:0;f30636bb2128:41943 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-07T11:48:09,957 DEBUG [RS_CLOSE_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1730980087152,,1730980087156.0dcd4b66301e80bbdcdef0c188d6d908. after waiting 0 ms 2024-11-07T11:48:09,957 INFO [RS:0;f30636bb2128:41943 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-07T11:48:09,957 INFO [RS:0;f30636bb2128:41943 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-07T11:48:09,957 INFO [RS:0;f30636bb2128:41943 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-07T11:48:09,957 DEBUG [RS_CLOSE_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1730980087152,,1730980087156.0dcd4b66301e80bbdcdef0c188d6d908. 2024-11-07T11:48:09,957 INFO [RS:0;f30636bb2128:41943 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-07T11:48:09,957 DEBUG [RS:0;f30636bb2128:41943 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 0dcd4b66301e80bbdcdef0c188d6d908=testBlockEvictionOnGracefulStop1730980087152,,1730980087156.0dcd4b66301e80bbdcdef0c188d6d908.} 2024-11-07T11:48:09,957 DEBUG [RS:0;f30636bb2128:41943 {}] regionserver.HRegionServer(1351): Waiting on 0dcd4b66301e80bbdcdef0c188d6d908, 1588230740 2024-11-07T11:48:09,957 INFO [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-07T11:48:09,958 DEBUG [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-07T11:48:09,958 INFO [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-07T11:48:09,958 DEBUG [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-07T11:48:09,958 DEBUG [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-07T11:48:09,958 DEBUG [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-07T11:48:09,958 DEBUG [StoreFileCloser-0dcd4b66301e80bbdcdef0c188d6d908-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/default/testBlockEvictionOnGracefulStop1730980087152/0dcd4b66301e80bbdcdef0c188d6d908/family/8f5ff55ee500451a9c507cbeb05bd8dd 2024-11-07T11:48:09,958 INFO [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.96 KB heapSize=5.58 KB 2024-11-07T11:48:09,959 DEBUG [RS:2;f30636bb2128:39281 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/oldWALs 2024-11-07T11:48:09,959 INFO [RS:2;f30636bb2128:39281 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f30636bb2128%2C39281%2C1730980089345:(num 1730980089539) 2024-11-07T11:48:09,959 DEBUG [RS:2;f30636bb2128:39281 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T11:48:09,959 INFO [RS:2;f30636bb2128:39281 {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T11:48:09,959 INFO [RS:2;f30636bb2128:39281 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-07T11:48:09,959 INFO [RS:2;f30636bb2128:39281 {}] hbase.ChoreService(370): Chore service for: regionserver/f30636bb2128:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-07T11:48:09,959 INFO [RS:2;f30636bb2128:39281 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-07T11:48:09,959 INFO [RS:2;f30636bb2128:39281 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-07T11:48:09,959 INFO [RS:2;f30636bb2128:39281 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-07T11:48:09,959 INFO [RS:2;f30636bb2128:39281 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-07T11:48:09,960 INFO [RS:2;f30636bb2128:39281 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39281 2024-11-07T11:48:09,962 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35521-0x1001a120c020000, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-07T11:48:09,962 INFO [RS:2;f30636bb2128:39281 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-07T11:48:09,962 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39281-0x1001a120c020003, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f30636bb2128,39281,1730980089345 2024-11-07T11:48:09,964 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f30636bb2128,39281,1730980089345] 2024-11-07T11:48:09,964 INFO [regionserver/f30636bb2128:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-07T11:48:09,966 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f30636bb2128,39281,1730980089345 already deleted, retry=false 2024-11-07T11:48:09,966 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; f30636bb2128,39281,1730980089345 expired; onlineServers=1 2024-11-07T11:48:09,967 DEBUG [RS_CLOSE_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/default/testBlockEvictionOnGracefulStop1730980087152/0dcd4b66301e80bbdcdef0c188d6d908/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-07T11:48:09,967 INFO [RS_CLOSE_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1730980087152,,1730980087156.0dcd4b66301e80bbdcdef0c188d6d908. 2024-11-07T11:48:09,967 DEBUG [RS_CLOSE_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0dcd4b66301e80bbdcdef0c188d6d908: Waiting for close lock at 1730980089956Running coprocessor pre-close hooks at 1730980089957 (+1 ms)Disabling compacts and flushes for region at 1730980089957Disabling writes for close at 1730980089957Writing region close event to WAL at 1730980089958 (+1 ms)Running coprocessor post-close hooks at 1730980089967 (+9 ms)Closed at 1730980089967 2024-11-07T11:48:09,967 DEBUG [RS_CLOSE_REGION-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1730980087152,,1730980087156.0dcd4b66301e80bbdcdef0c188d6d908. 2024-11-07T11:48:09,975 DEBUG [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 739ebc4292c746dcbebd57f16de524ba 2024-11-07T11:48:09,989 INFO [regionserver/f30636bb2128:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T11:48:09,992 DEBUG [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740/.tmp/info/739ebc4292c746dcbebd57f16de524ba is 209, key is testBlockEvictionOnGracefulStop1730980087152,,1730980087156.0dcd4b66301e80bbdcdef0c188d6d908./info:regioninfo/1730980089394/Put/seqid=0 2024-11-07T11:48:09,993 INFO [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.74 KB at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740/.tmp/info/739ebc4292c746dcbebd57f16de524ba 2024-11-07T11:48:10,002 INFO [regionserver/f30636bb2128:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T11:48:10,003 DEBUG [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 1f2c1af14d3044989dab0bbed7ed8a50 2024-11-07T11:48:10,022 DEBUG [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740/.tmp/ns/1f2c1af14d3044989dab0bbed7ed8a50 is 43, key is default/ns:d/1730980087112/Put/seqid=0 2024-11-07T11:48:10,023 INFO [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740/.tmp/ns/1f2c1af14d3044989dab0bbed7ed8a50 2024-11-07T11:48:10,033 DEBUG [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 054489320ff143348c056e6f15b19c8e 2024-11-07T11:48:10,046 DEBUG [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740/.tmp/table/054489320ff143348c056e6f15b19c8e is 80, key is testBlockEvictionOnGracefulStop1730980087152/table:state/1730980087531/Put/seqid=0 2024-11-07T11:48:10,046 INFO [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740/.tmp/table/054489320ff143348c056e6f15b19c8e 2024-11-07T11:48:10,049 DEBUG [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740/.tmp/info/739ebc4292c746dcbebd57f16de524ba as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740/info/739ebc4292c746dcbebd57f16de524ba 2024-11-07T11:48:10,050 DEBUG [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740/info/739ebc4292c746dcbebd57f16de524ba, delay=958 ms 2024-11-07T11:48:10,051 INFO [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740/info/739ebc4292c746dcbebd57f16de524ba, entries=14, sequenceid=13, filesize=7.5 K 2024-11-07T11:48:10,052 DEBUG [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740/.tmp/ns/1f2c1af14d3044989dab0bbed7ed8a50 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740/ns/1f2c1af14d3044989dab0bbed7ed8a50 2024-11-07T11:48:10,053 DEBUG [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740/ns/1f2c1af14d3044989dab0bbed7ed8a50, delay=957 ms 2024-11-07T11:48:10,053 INFO [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740/ns/1f2c1af14d3044989dab0bbed7ed8a50, entries=2, sequenceid=13, filesize=5.0 K 2024-11-07T11:48:10,054 DEBUG [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740/.tmp/table/054489320ff143348c056e6f15b19c8e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740/table/054489320ff143348c056e6f15b19c8e 2024-11-07T11:48:10,056 DEBUG [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740/table/054489320ff143348c056e6f15b19c8e, delay=938 ms 2024-11-07T11:48:10,057 INFO [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740/table/054489320ff143348c056e6f15b19c8e, entries=2, sequenceid=13, filesize=5.3 K 2024-11-07T11:48:10,058 INFO [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 100ms, sequenceid=13, compaction requested=false 2024-11-07T11:48:10,059 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740/info/739ebc4292c746dcbebd57f16de524ba 2024-11-07T11:48:10,059 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740/ns/1f2c1af14d3044989dab0bbed7ed8a50 2024-11-07T11:48:10,060 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740/table/054489320ff143348c056e6f15b19c8e 2024-11-07T11:48:10,065 DEBUG [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/data/hbase/meta/1588230740/recovered.edits/16.seqid, newMaxSeqId=16, maxSeqId=1 2024-11-07T11:48:10,065 DEBUG [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-07T11:48:10,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39281-0x1001a120c020003, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T11:48:10,065 INFO [RS:2;f30636bb2128:39281 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-07T11:48:10,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39281-0x1001a120c020003, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T11:48:10,065 INFO [RS:2;f30636bb2128:39281 {}] regionserver.HRegionServer(1031): Exiting; stopping=f30636bb2128,39281,1730980089345; zookeeper connection closed. 2024-11-07T11:48:10,065 INFO [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-07T11:48:10,065 DEBUG [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1730980089957Running coprocessor pre-close hooks at 1730980089958 (+1 ms)Disabling compacts and flushes for region at 1730980089958Disabling writes for close at 1730980089958Obtaining lock to block concurrent updates at 1730980089958Preparing flush snapshotting stores in 1588230740 at 1730980089958Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3035, getHeapSize=5648, getOffHeapSize=0, getCellsCount=22 at 1730980089959 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1730980089959Flushing 1588230740/info: creating writer at 1730980089959Flushing 1588230740/info: appending metadata at 1730980089992 (+33 ms)Flushing 1588230740/info: closing flushed file at 1730980089992Flushing 1588230740/ns: creating writer at 1730980089996 (+4 ms)Flushing 1588230740/ns: appending metadata at 1730980090021 (+25 ms)Flushing 1588230740/ns: closing flushed file at 1730980090021Flushing 1588230740/table: creating writer at 1730980090026 (+5 ms)Flushing 1588230740/table: appending metadata at 1730980090045 (+19 ms)Flushing 1588230740/table: closing flushed file at 1730980090045Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@496bc0d3: reopening flushed file at 1730980090048 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c1da750: reopening flushed file at 1730980090051 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@333654e: reopening flushed file at 1730980090054 (+3 ms)Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 100ms, sequenceid=13, compaction requested=false at 1730980090058 (+4 ms)Writing region close event to WAL at 1730980090060 (+2 ms)Running coprocessor post-close hooks at 1730980090065 (+5 ms)Closed at 1730980090065 2024-11-07T11:48:10,065 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3c8a537 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3c8a537 2024-11-07T11:48:10,065 DEBUG [RS_CLOSE_META-regionserver/f30636bb2128:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-07T11:48:10,158 INFO [RS:0;f30636bb2128:41943 {}] regionserver.HRegionServer(976): stopping server f30636bb2128,41943,1730980086188; all regions closed. 2024-11-07T11:48:10,160 DEBUG [RS:0;f30636bb2128:41943 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/oldWALs 2024-11-07T11:48:10,160 INFO [RS:0;f30636bb2128:41943 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f30636bb2128%2C41943%2C1730980086188.meta:.meta(num 1730980087062) 2024-11-07T11:48:10,162 DEBUG [RS:0;f30636bb2128:41943 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/oldWALs 2024-11-07T11:48:10,162 INFO [RS:0;f30636bb2128:41943 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f30636bb2128%2C41943%2C1730980086188:(num 1730980086729) 2024-11-07T11:48:10,162 DEBUG [RS:0;f30636bb2128:41943 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T11:48:10,162 INFO [RS:0;f30636bb2128:41943 {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T11:48:10,162 INFO [RS:0;f30636bb2128:41943 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-07T11:48:10,162 INFO [RS:0;f30636bb2128:41943 {}] hbase.ChoreService(370): Chore service for: regionserver/f30636bb2128:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-07T11:48:10,162 INFO [RS:0;f30636bb2128:41943 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-07T11:48:10,162 INFO [regionserver/f30636bb2128:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-07T11:48:10,163 INFO [RS:0;f30636bb2128:41943 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41943 2024-11-07T11:48:10,165 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41943-0x1001a120c020001, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f30636bb2128,41943,1730980086188 2024-11-07T11:48:10,165 INFO [RS:0;f30636bb2128:41943 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-07T11:48:10,165 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35521-0x1001a120c020000, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-07T11:48:10,167 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f30636bb2128,41943,1730980086188] 2024-11-07T11:48:10,168 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f30636bb2128,41943,1730980086188 already deleted, retry=false 2024-11-07T11:48:10,169 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; f30636bb2128,41943,1730980086188 expired; onlineServers=0 2024-11-07T11:48:10,169 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'f30636bb2128,35521,1730980086130' ***** 2024-11-07T11:48:10,169 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-07T11:48:10,169 INFO [M:0;f30636bb2128:35521 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-07T11:48:10,169 INFO [M:0;f30636bb2128:35521 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-07T11:48:10,169 DEBUG [M:0;f30636bb2128:35521 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-07T11:48:10,169 DEBUG [M:0;f30636bb2128:35521 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-07T11:48:10,169 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-07T11:48:10,169 DEBUG [master/f30636bb2128:0:becomeActiveMaster-HFileCleaner.large.0-1730980086448 {}] cleaner.HFileCleaner(306): Exit Thread[master/f30636bb2128:0:becomeActiveMaster-HFileCleaner.large.0-1730980086448,5,FailOnTimeoutGroup] 2024-11-07T11:48:10,169 DEBUG [master/f30636bb2128:0:becomeActiveMaster-HFileCleaner.small.0-1730980086448 {}] cleaner.HFileCleaner(306): Exit Thread[master/f30636bb2128:0:becomeActiveMaster-HFileCleaner.small.0-1730980086448,5,FailOnTimeoutGroup] 2024-11-07T11:48:10,169 INFO [M:0;f30636bb2128:35521 {}] hbase.ChoreService(370): Chore service for: master/f30636bb2128:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-07T11:48:10,170 INFO [M:0;f30636bb2128:35521 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-07T11:48:10,170 DEBUG [M:0;f30636bb2128:35521 {}] master.HMaster(1795): Stopping service threads 2024-11-07T11:48:10,170 INFO [M:0;f30636bb2128:35521 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-07T11:48:10,170 INFO [M:0;f30636bb2128:35521 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T11:48:10,170 INFO [M:0;f30636bb2128:35521 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T11:48:10,170 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-07T11:48:10,171 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35521-0x1001a120c020000, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-07T11:48:10,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35521-0x1001a120c020000, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T11:48:10,172 DEBUG [M:0;f30636bb2128:35521 {}] zookeeper.ZKUtil(347): master:35521-0x1001a120c020000, quorum=127.0.0.1:63417, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-07T11:48:10,172 WARN [M:0;f30636bb2128:35521 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-07T11:48:10,172 INFO [M:0;f30636bb2128:35521 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/.lastflushedseqids 2024-11-07T11:48:10,179 INFO [M:0;f30636bb2128:35521 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T11:48:10,179 INFO [M:0;f30636bb2128:35521 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-07T11:48:10,179 DEBUG [M:0;f30636bb2128:35521 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-07T11:48:10,179 INFO [M:0;f30636bb2128:35521 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T11:48:10,179 DEBUG [M:0;f30636bb2128:35521 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T11:48:10,179 DEBUG [M:0;f30636bb2128:35521 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-07T11:48:10,179 DEBUG [M:0;f30636bb2128:35521 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T11:48:10,179 INFO [M:0;f30636bb2128:35521 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=32.83 KB heapSize=41.08 KB 2024-11-07T11:48:10,190 DEBUG [M:0;f30636bb2128:35521 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 115a6d5569574887a19530ffaa2ac354 2024-11-07T11:48:10,203 DEBUG [M:0;f30636bb2128:35521 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/115a6d5569574887a19530ffaa2ac354 is 82, key is hbase:meta,,1/info:regioninfo/1730980087095/Put/seqid=0 2024-11-07T11:48:10,204 INFO [M:0;f30636bb2128:35521 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/115a6d5569574887a19530ffaa2ac354 2024-11-07T11:48:10,212 DEBUG [M:0;f30636bb2128:35521 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 41cd4562106c47eb976408b9372e4f6d 2024-11-07T11:48:10,225 DEBUG [M:0;f30636bb2128:35521 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/41cd4562106c47eb976408b9372e4f6d is 782, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1730980087535/Put/seqid=0 2024-11-07T11:48:10,225 INFO [M:0;f30636bb2128:35521 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.09 KB at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/41cd4562106c47eb976408b9372e4f6d 2024-11-07T11:48:10,233 DEBUG [M:0;f30636bb2128:35521 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on e38c4cc0735841888cd8b06ba1aa8194 2024-11-07T11:48:10,246 DEBUG [M:0;f30636bb2128:35521 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e38c4cc0735841888cd8b06ba1aa8194 is 69, key is f30636bb2128,39281,1730980089345/rs:state/1730980089383/Put/seqid=0 2024-11-07T11:48:10,247 INFO [M:0;f30636bb2128:35521 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e38c4cc0735841888cd8b06ba1aa8194 2024-11-07T11:48:10,249 INFO [M:0;f30636bb2128:35521 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e38c4cc0735841888cd8b06ba1aa8194 2024-11-07T11:48:10,249 DEBUG [M:0;f30636bb2128:35521 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/115a6d5569574887a19530ffaa2ac354 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/115a6d5569574887a19530ffaa2ac354 2024-11-07T11:48:10,251 INFO [M:0;f30636bb2128:35521 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/115a6d5569574887a19530ffaa2ac354, entries=8, sequenceid=83, filesize=5.5 K 2024-11-07T11:48:10,251 DEBUG [M:0;f30636bb2128:35521 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/41cd4562106c47eb976408b9372e4f6d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/41cd4562106c47eb976408b9372e4f6d 2024-11-07T11:48:10,253 INFO [M:0;f30636bb2128:35521 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/41cd4562106c47eb976408b9372e4f6d, entries=9, sequenceid=83, filesize=6.4 K 2024-11-07T11:48:10,253 DEBUG [M:0;f30636bb2128:35521 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e38c4cc0735841888cd8b06ba1aa8194 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e38c4cc0735841888cd8b06ba1aa8194 2024-11-07T11:48:10,254 INFO [M:0;f30636bb2128:35521 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e38c4cc0735841888cd8b06ba1aa8194 2024-11-07T11:48:10,254 INFO [M:0;f30636bb2128:35521 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d4145358-6579-f2ce-9998-da30df729a9e/4bb8c7d6-089c-42a6-8fa6-61137a4d8a47/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e38c4cc0735841888cd8b06ba1aa8194, entries=3, sequenceid=83, filesize=5.3 K 2024-11-07T11:48:10,255 INFO [M:0;f30636bb2128:35521 {}] regionserver.HRegion(3140): Finished flush of dataSize ~32.83 KB/33614, heapSize ~40.78 KB/41760, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 76ms, sequenceid=83, compaction requested=false 2024-11-07T11:48:10,257 INFO [M:0;f30636bb2128:35521 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T11:48:10,257 DEBUG [M:0;f30636bb2128:35521 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1730980090179Disabling compacts and flushes for region at 1730980090179Disabling writes for close at 1730980090179Obtaining lock to block concurrent updates at 1730980090179Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1730980090179Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=33614, getHeapSize=42000, getOffHeapSize=0, getCellsCount=98 at 1730980090180 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1730980090180Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1730980090180Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1730980090203 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1730980090203Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1730980090206 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1730980090224 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1730980090224Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1730980090227 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1730980090246 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1730980090246Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3a9eea8c: reopening flushed file at 1730980090249 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f29cf8f: reopening flushed file at 1730980090251 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4627110f: reopening flushed file at 1730980090253 (+2 ms)Finished flush of dataSize ~32.83 KB/33614, heapSize ~40.78 KB/41760, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 76ms, sequenceid=83, compaction requested=false at 1730980090255 (+2 ms)Writing region close event to WAL at 1730980090257 (+2 ms)Closed at 1730980090257 2024-11-07T11:48:10,258 INFO [M:0;f30636bb2128:35521 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-07T11:48:10,258 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-07T11:48:10,258 INFO [M:0;f30636bb2128:35521 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35521 2024-11-07T11:48:10,258 INFO [M:0;f30636bb2128:35521 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-07T11:48:10,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41943-0x1001a120c020001, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T11:48:10,267 INFO [RS:0;f30636bb2128:41943 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-07T11:48:10,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41943-0x1001a120c020001, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T11:48:10,267 INFO [RS:0;f30636bb2128:41943 {}] regionserver.HRegionServer(1031): Exiting; stopping=f30636bb2128,41943,1730980086188; zookeeper connection closed. 2024-11-07T11:48:10,268 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@748b0191 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@748b0191 2024-11-07T11:48:10,268 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-07T11:48:10,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35521-0x1001a120c020000, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T11:48:10,360 INFO [M:0;f30636bb2128:35521 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-07T11:48:10,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35521-0x1001a120c020000, quorum=127.0.0.1:63417, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T11:48:10,363 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-07T11:48:10,364 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-07T11:48:10,364 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T11:48:10,373 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=72 (was 51) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1905 (was 1905), ProcessCount=11 (was 11), AvailableMemoryMB=2855 (was 2919)