2024-11-16 05:54:58,075 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-16 05:54:58,098 main DEBUG Took 0.021740 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-16 05:54:58,099 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-16 05:54:58,099 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-16 05:54:58,100 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-16 05:54:58,101 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:54:58,107 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-16 05:54:58,127 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:54:58,130 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:54:58,130 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:54:58,131 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:54:58,131 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:54:58,131 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:54:58,132 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:54:58,132 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:54:58,133 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:54:58,133 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:54:58,134 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:54:58,134 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:54:58,135 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:54:58,135 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:54:58,135 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:54:58,136 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:54:58,136 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:54:58,136 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:54:58,137 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:54:58,137 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:54:58,138 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:54:58,138 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:54:58,139 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:54:58,140 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:54:58,140 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:54:58,140 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-16 05:54:58,142 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:54:58,144 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-16 05:54:58,146 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-16 05:54:58,147 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-16 05:54:58,149 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-16 05:54:58,150 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-16 05:54:58,157 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-16 05:54:58,160 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-16 05:54:58,162 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-16 05:54:58,162 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-16 05:54:58,163 main DEBUG createAppenders(={Console}) 2024-11-16 05:54:58,164 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-16 05:54:58,164 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-16 05:54:58,165 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-16 05:54:58,165 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-16 05:54:58,165 main DEBUG OutputStream closed 2024-11-16 05:54:58,166 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-16 05:54:58,166 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-16 05:54:58,166 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-16 05:54:58,259 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-16 05:54:58,262 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-16 05:54:58,263 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-16 05:54:58,264 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-16 05:54:58,269 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-16 05:54:58,270 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-16 05:54:58,270 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-16 05:54:58,271 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-16 05:54:58,271 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-16 05:54:58,271 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-16 05:54:58,272 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-16 05:54:58,272 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-16 05:54:58,273 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-16 05:54:58,273 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-16 05:54:58,273 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-16 05:54:58,274 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-16 05:54:58,274 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-16 05:54:58,275 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-16 05:54:58,279 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-16 05:54:58,279 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-16 05:54:58,280 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-16 05:54:58,281 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-16T05:54:58,299 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-16 05:54:58,304 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-16 05:54:58,304 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-16T05:54:58,621 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763 2024-11-16T05:54:58,656 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=235, ProcessCount=13, AvailableMemoryMB=5064 2024-11-16T05:54:58,779 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-16T05:54:58,901 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T05:54:58,926 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/cluster_68e22f93-7cf1-a4c3-3363-105931cfa919, deleteOnExit=true 2024-11-16T05:54:59,026 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/cluster_68e22f93-7cf1-a4c3-3363-105931cfa919/zookeeper_0, clientPort=62345, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/cluster_68e22f93-7cf1-a4c3-3363-105931cfa919/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/cluster_68e22f93-7cf1-a4c3-3363-105931cfa919/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-16T05:54:59,039 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62345 2024-11-16T05:54:59,040 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T05:54:59,159 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926 with version=8 2024-11-16T05:54:59,160 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/hbase-staging 2024-11-16T05:54:59,285 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-16T05:54:59,601 INFO [Time-limited test {}] client.ConnectionUtils(128): master/3456ee6a3164:0 server-side Connection retries=45 2024-11-16T05:54:59,614 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T05:54:59,616 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T05:54:59,627 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T05:54:59,627 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T05:54:59,628 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T05:54:59,816 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-16T05:54:59,922 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T05:54:59,968 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-16T05:54:59,978 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-16T05:54:59,982 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T05:55:00,031 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 90799 (auto-detected) 2024-11-16T05:55:00,033 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-16T05:55:00,062 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38307 2024-11-16T05:55:00,093 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38307 connecting to ZooKeeper ensemble=127.0.0.1:62345 2024-11-16T05:55:00,139 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:383070x0, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T05:55:00,142 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38307-0x10047194fa70000 connected 2024-11-16T05:55:00,181 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T05:55:00,183 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T05:55:00,194 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38307-0x10047194fa70000, quorum=127.0.0.1:62345, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T05:55:00,199 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926, hbase.cluster.distributed=false 2024-11-16T05:55:00,229 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38307-0x10047194fa70000, quorum=127.0.0.1:62345, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T05:55:00,241 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38307 2024-11-16T05:55:00,242 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38307 2024-11-16T05:55:00,242 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38307 2024-11-16T05:55:00,245 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38307 2024-11-16T05:55:00,251 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38307 2024-11-16T05:55:00,337 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/3456ee6a3164:0 server-side Connection retries=45 2024-11-16T05:55:00,338 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T05:55:00,339 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T05:55:00,339 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T05:55:00,339 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T05:55:00,339 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T05:55:00,342 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T05:55:00,345 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T05:55:00,346 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33069 2024-11-16T05:55:00,349 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33069 connecting to ZooKeeper ensemble=127.0.0.1:62345 2024-11-16T05:55:00,349 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T05:55:00,350 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T05:55:00,355 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:330690x0, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T05:55:00,358 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33069-0x10047194fa70001 connected 2024-11-16T05:55:00,359 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33069-0x10047194fa70001, quorum=127.0.0.1:62345, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T05:55:00,367 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T05:55:00,384 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/bucket.cache 2024-11-16T05:55:00,395 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-16T05:55:00,398 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-16T05:55:00,400 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-16T05:55:00,407 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-16T05:55:00,407 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-16T05:55:00,420 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-16T05:55:00,421 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/bucket.persistence is complete. 2024-11-16T05:55:00,428 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-16T05:55:00,436 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T05:55:00,439 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33069-0x10047194fa70001, quorum=127.0.0.1:62345, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T05:55:00,445 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33069-0x10047194fa70001, quorum=127.0.0.1:62345, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T05:55:00,450 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33069 2024-11-16T05:55:00,451 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33069 2024-11-16T05:55:00,451 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33069 2024-11-16T05:55:00,453 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33069 2024-11-16T05:55:00,457 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33069 2024-11-16T05:55:00,482 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/3456ee6a3164:0 server-side Connection retries=45 2024-11-16T05:55:00,483 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T05:55:00,483 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T05:55:00,483 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T05:55:00,484 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T05:55:00,484 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T05:55:00,484 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T05:55:00,485 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T05:55:00,486 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36505 2024-11-16T05:55:00,489 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36505 connecting to ZooKeeper ensemble=127.0.0.1:62345 2024-11-16T05:55:00,490 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T05:55:00,492 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T05:55:00,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:365050x0, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T05:55:00,498 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:365050x0, quorum=127.0.0.1:62345, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T05:55:00,499 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T05:55:00,506 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36505-0x10047194fa70002 connected 2024-11-16T05:55:00,507 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/bucket.cache 2024-11-16T05:55:00,507 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-16T05:55:00,507 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-16T05:55:00,508 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-16T05:55:00,514 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-16T05:55:00,514 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-16T05:55:00,515 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-16T05:55:00,516 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-16T05:55:00,517 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/bucket.persistence is complete. 2024-11-16T05:55:00,517 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T05:55:00,519 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36505-0x10047194fa70002, quorum=127.0.0.1:62345, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T05:55:00,525 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36505-0x10047194fa70002, quorum=127.0.0.1:62345, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T05:55:00,530 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36505 2024-11-16T05:55:00,530 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36505 2024-11-16T05:55:00,535 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36505 2024-11-16T05:55:00,540 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36505 2024-11-16T05:55:00,541 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36505 2024-11-16T05:55:00,573 DEBUG [M:0;3456ee6a3164:38307 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;3456ee6a3164:38307 2024-11-16T05:55:00,575 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/3456ee6a3164,38307,1731736499374 2024-11-16T05:55:00,585 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38307-0x10047194fa70000, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T05:55:00,585 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33069-0x10047194fa70001, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T05:55:00,585 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36505-0x10047194fa70002, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T05:55:00,594 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38307-0x10047194fa70000, quorum=127.0.0.1:62345, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/3456ee6a3164,38307,1731736499374 2024-11-16T05:55:00,626 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38307-0x10047194fa70000, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:55:00,626 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33069-0x10047194fa70001, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T05:55:00,626 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33069-0x10047194fa70001, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:55:00,626 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36505-0x10047194fa70002, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T05:55:00,626 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36505-0x10047194fa70002, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:55:00,627 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38307-0x10047194fa70000, quorum=127.0.0.1:62345, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-16T05:55:00,629 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/3456ee6a3164,38307,1731736499374 from backup master directory 2024-11-16T05:55:00,632 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38307-0x10047194fa70000, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/3456ee6a3164,38307,1731736499374 2024-11-16T05:55:00,632 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36505-0x10047194fa70002, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T05:55:00,632 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33069-0x10047194fa70001, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T05:55:00,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38307-0x10047194fa70000, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T05:55:00,634 WARN [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T05:55:00,634 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=3456ee6a3164,38307,1731736499374 2024-11-16T05:55:00,637 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-16T05:55:00,638 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-16T05:55:00,647 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/hbase.id] with ID: 4f9cc7e2-2fa1-4d4f-9f0f-9a4f2fa1dbe2 2024-11-16T05:55:00,647 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/.tmp/hbase.id 2024-11-16T05:55:00,655 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/hbase.id] 2024-11-16T05:55:00,693 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T05:55:00,699 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-16T05:55:00,718 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-16T05:55:00,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33069-0x10047194fa70001, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:55:00,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36505-0x10047194fa70002, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:55:00,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38307-0x10047194fa70000, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:55:00,748 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T05:55:00,751 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-16T05:55:00,788 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:55:00,793 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T05:55:00,847 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/MasterData/data/master/store 2024-11-16T05:55:00,867 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-16T05:55:00,870 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:55:00,871 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T05:55:00,872 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T05:55:00,872 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T05:55:00,873 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T05:55:00,874 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T05:55:00,874 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T05:55:00,875 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731736500871Disabling compacts and flushes for region at 1731736500871Disabling writes for close at 1731736500874 (+3 ms)Writing region close event to WAL at 1731736500874Closed at 1731736500874 2024-11-16T05:55:00,875 WARN [master/3456ee6a3164:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/MasterData/data/master/store/.initializing 2024-11-16T05:55:00,875 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/MasterData/WALs/3456ee6a3164,38307,1731736499374 2024-11-16T05:55:00,888 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T05:55:00,901 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3456ee6a3164%2C38307%2C1731736499374, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/MasterData/WALs/3456ee6a3164,38307,1731736499374, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/MasterData/oldWALs, maxLogs=10 2024-11-16T05:55:00,934 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/MasterData/WALs/3456ee6a3164,38307,1731736499374/3456ee6a3164%2C38307%2C1731736499374.1731736500905 2024-11-16T05:55:00,935 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-16T05:55:00,936 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-16T05:55:00,936 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:55:00,939 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:55:00,940 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:55:00,976 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:55:01,018 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-16T05:55:01,024 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:55:01,026 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:55:01,027 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:55:01,033 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-16T05:55:01,033 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:55:01,034 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:55:01,034 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:55:01,040 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-16T05:55:01,040 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:55:01,041 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:55:01,042 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:55:01,047 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-16T05:55:01,047 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:55:01,048 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:55:01,049 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:55:01,053 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:55:01,054 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:55:01,058 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:55:01,059 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:55:01,063 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T05:55:01,066 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:55:01,074 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:55:01,075 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70661923, jitterRate=0.05294470489025116}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T05:55:01,086 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731736500952Initializing all the Stores at 1731736500952Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731736500953 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736500954 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736500954Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736500954Cleaning up temporary data from old regions at 1731736501060 (+106 ms)Region opened successfully at 1731736501086 (+26 ms) 2024-11-16T05:55:01,088 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-16T05:55:01,132 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@508a9b6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3456ee6a3164/172.17.0.2:0 2024-11-16T05:55:01,175 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-16T05:55:01,190 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-16T05:55:01,190 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-16T05:55:01,195 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T05:55:01,196 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-16T05:55:01,202 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-16T05:55:01,202 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-16T05:55:01,228 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-16T05:55:01,236 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38307-0x10047194fa70000, quorum=127.0.0.1:62345, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-16T05:55:01,239 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-16T05:55:01,241 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-16T05:55:01,243 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38307-0x10047194fa70000, quorum=127.0.0.1:62345, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-16T05:55:01,244 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-16T05:55:01,247 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-16T05:55:01,251 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38307-0x10047194fa70000, quorum=127.0.0.1:62345, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-16T05:55:01,253 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-16T05:55:01,255 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38307-0x10047194fa70000, quorum=127.0.0.1:62345, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-16T05:55:01,258 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-16T05:55:01,275 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38307-0x10047194fa70000, quorum=127.0.0.1:62345, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-16T05:55:01,276 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-16T05:55:01,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33069-0x10047194fa70001, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T05:55:01,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36505-0x10047194fa70002, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T05:55:01,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33069-0x10047194fa70001, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:55:01,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36505-0x10047194fa70002, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:55:01,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38307-0x10047194fa70000, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T05:55:01,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38307-0x10047194fa70000, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:55:01,293 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=3456ee6a3164,38307,1731736499374, sessionid=0x10047194fa70000, setting cluster-up flag (Was=false) 2024-11-16T05:55:01,307 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38307-0x10047194fa70000, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:55:01,307 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33069-0x10047194fa70001, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:55:01,307 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36505-0x10047194fa70002, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:55:01,312 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-16T05:55:01,314 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=3456ee6a3164,38307,1731736499374 2024-11-16T05:55:01,320 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38307-0x10047194fa70000, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:55:01,320 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33069-0x10047194fa70001, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:55:01,320 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36505-0x10047194fa70002, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:55:01,326 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-16T05:55:01,328 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=3456ee6a3164,38307,1731736499374 2024-11-16T05:55:01,332 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-16T05:55:01,346 INFO [RS:0;3456ee6a3164:33069 {}] regionserver.HRegionServer(746): ClusterId : 4f9cc7e2-2fa1-4d4f-9f0f-9a4f2fa1dbe2 2024-11-16T05:55:01,346 INFO [RS:1;3456ee6a3164:36505 {}] regionserver.HRegionServer(746): ClusterId : 4f9cc7e2-2fa1-4d4f-9f0f-9a4f2fa1dbe2 2024-11-16T05:55:01,349 DEBUG [RS:0;3456ee6a3164:33069 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T05:55:01,349 DEBUG [RS:1;3456ee6a3164:36505 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T05:55:01,354 DEBUG [RS:0;3456ee6a3164:33069 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T05:55:01,354 DEBUG [RS:1;3456ee6a3164:36505 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T05:55:01,354 DEBUG [RS:0;3456ee6a3164:33069 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T05:55:01,355 DEBUG [RS:1;3456ee6a3164:36505 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T05:55:01,362 DEBUG [RS:0;3456ee6a3164:33069 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T05:55:01,362 DEBUG [RS:1;3456ee6a3164:36505 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T05:55:01,363 DEBUG [RS:0;3456ee6a3164:33069 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a5bc1fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3456ee6a3164/172.17.0.2:0 2024-11-16T05:55:01,363 DEBUG [RS:1;3456ee6a3164:36505 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2afdf127, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3456ee6a3164/172.17.0.2:0 2024-11-16T05:55:01,377 DEBUG [RS:0;3456ee6a3164:33069 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;3456ee6a3164:33069 2024-11-16T05:55:01,382 DEBUG [RS:1;3456ee6a3164:36505 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;3456ee6a3164:36505 2024-11-16T05:55:01,380 INFO [RS:0;3456ee6a3164:33069 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T05:55:01,384 INFO [RS:0;3456ee6a3164:33069 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T05:55:01,384 DEBUG [RS:0;3456ee6a3164:33069 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T05:55:01,385 INFO [RS:1;3456ee6a3164:36505 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T05:55:01,385 INFO [RS:1;3456ee6a3164:36505 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T05:55:01,385 DEBUG [RS:1;3456ee6a3164:36505 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T05:55:01,387 INFO [RS:1;3456ee6a3164:36505 {}] regionserver.HRegionServer(2659): reportForDuty to master=3456ee6a3164,38307,1731736499374 with port=36505, startcode=1731736500481 2024-11-16T05:55:01,387 INFO [RS:0;3456ee6a3164:33069 {}] regionserver.HRegionServer(2659): reportForDuty to master=3456ee6a3164,38307,1731736499374 with port=33069, startcode=1731736500303 2024-11-16T05:55:01,399 DEBUG [RS:0;3456ee6a3164:33069 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T05:55:01,399 DEBUG [RS:1;3456ee6a3164:36505 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T05:55:01,421 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T05:55:01,435 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-16T05:55:01,446 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-16T05:55:01,458 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 3456ee6a3164,38307,1731736499374 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-16T05:55:01,468 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/3456ee6a3164:0, corePoolSize=5, maxPoolSize=5 2024-11-16T05:55:01,469 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/3456ee6a3164:0, corePoolSize=5, maxPoolSize=5 2024-11-16T05:55:01,469 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/3456ee6a3164:0, corePoolSize=5, maxPoolSize=5 2024-11-16T05:55:01,469 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/3456ee6a3164:0, corePoolSize=5, maxPoolSize=5 2024-11-16T05:55:01,469 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/3456ee6a3164:0, corePoolSize=10, maxPoolSize=10 2024-11-16T05:55:01,470 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:01,470 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/3456ee6a3164:0, corePoolSize=2, maxPoolSize=2 2024-11-16T05:55:01,470 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:01,480 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53133, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T05:55:01,483 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T05:55:01,483 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55957, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T05:55:01,484 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-16T05:55:01,488 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38307 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:55:01,489 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:55:01,489 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-16T05:55:01,493 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731736531492 2024-11-16T05:55:01,495 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-16T05:55:01,496 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-16T05:55:01,496 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38307 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:55:01,500 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-16T05:55:01,500 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-16T05:55:01,501 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-16T05:55:01,501 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-16T05:55:01,502 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:01,510 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-16T05:55:01,511 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-16T05:55:01,512 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-16T05:55:01,515 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-16T05:55:01,516 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-16T05:55:01,526 DEBUG [RS:0;3456ee6a3164:33069 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-16T05:55:01,526 DEBUG [RS:1;3456ee6a3164:36505 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-16T05:55:01,526 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/3456ee6a3164:0:becomeActiveMaster-HFileCleaner.large.0-1731736501517,5,FailOnTimeoutGroup] 2024-11-16T05:55:01,526 WARN [RS:0;3456ee6a3164:33069 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-16T05:55:01,527 WARN [RS:1;3456ee6a3164:36505 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-16T05:55:01,529 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/3456ee6a3164:0:becomeActiveMaster-HFileCleaner.small.0-1731736501526,5,FailOnTimeoutGroup] 2024-11-16T05:55:01,529 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:01,530 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-16T05:55:01,531 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:01,532 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:01,544 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-16T05:55:01,545 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926 2024-11-16T05:55:01,555 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:55:01,556 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-16T05:55:01,561 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T05:55:01,561 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:55:01,562 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:55:01,562 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-16T05:55:01,570 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T05:55:01,570 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:55:01,570 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:55:01,571 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-16T05:55:01,579 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T05:55:01,579 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:55:01,579 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:55:01,580 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-16T05:55:01,584 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T05:55:01,584 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:55:01,585 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:55:01,585 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T05:55:01,587 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740 2024-11-16T05:55:01,587 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740 2024-11-16T05:55:01,588 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T05:55:01,588 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T05:55:01,589 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T05:55:01,590 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T05:55:01,598 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:55:01,598 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63423254, jitterRate=-0.05491986870765686}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T05:55:01,604 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731736501555Initializing all the Stores at 1731736501556 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731736501556Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731736501556Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736501556Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731736501556Cleaning up temporary data from old regions at 1731736501588 (+32 ms)Region opened successfully at 1731736501604 (+16 ms) 2024-11-16T05:55:01,605 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T05:55:01,605 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T05:55:01,605 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T05:55:01,605 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T05:55:01,605 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T05:55:01,607 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T05:55:01,607 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731736501604Disabling compacts and flushes for region at 1731736501604Disabling writes for close at 1731736501605 (+1 ms)Writing region close event to WAL at 1731736501606 (+1 ms)Closed at 1731736501607 (+1 ms) 2024-11-16T05:55:01,610 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T05:55:01,610 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-16T05:55:01,619 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-16T05:55:01,628 INFO [RS:0;3456ee6a3164:33069 {}] regionserver.HRegionServer(2659): reportForDuty to master=3456ee6a3164,38307,1731736499374 with port=33069, startcode=1731736500303 2024-11-16T05:55:01,628 INFO [RS:1;3456ee6a3164:36505 {}] regionserver.HRegionServer(2659): reportForDuty to master=3456ee6a3164,38307,1731736499374 with port=36505, startcode=1731736500481 2024-11-16T05:55:01,630 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38307 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 3456ee6a3164,33069,1731736500303 2024-11-16T05:55:01,633 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T05:55:01,633 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38307 {}] master.ServerManager(517): Registering regionserver=3456ee6a3164,33069,1731736500303 2024-11-16T05:55:01,636 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-16T05:55:01,642 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38307 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 3456ee6a3164,36505,1731736500481 2024-11-16T05:55:01,643 DEBUG [RS:0;3456ee6a3164:33069 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926 2024-11-16T05:55:01,643 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38307 {}] master.ServerManager(517): Registering regionserver=3456ee6a3164,36505,1731736500481 2024-11-16T05:55:01,643 DEBUG [RS:0;3456ee6a3164:33069 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-16T05:55:01,643 DEBUG [RS:0;3456ee6a3164:33069 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T05:55:01,646 DEBUG [RS:1;3456ee6a3164:36505 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926 2024-11-16T05:55:01,646 DEBUG [RS:1;3456ee6a3164:36505 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-16T05:55:01,646 DEBUG [RS:1;3456ee6a3164:36505 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T05:55:01,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38307-0x10047194fa70000, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T05:55:01,651 DEBUG [RS:0;3456ee6a3164:33069 {}] zookeeper.ZKUtil(111): regionserver:33069-0x10047194fa70001, quorum=127.0.0.1:62345, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/3456ee6a3164,33069,1731736500303 2024-11-16T05:55:01,651 WARN [RS:0;3456ee6a3164:33069 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T05:55:01,651 INFO [RS:0;3456ee6a3164:33069 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T05:55:01,652 DEBUG [RS:0;3456ee6a3164:33069 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/WALs/3456ee6a3164,33069,1731736500303 2024-11-16T05:55:01,653 DEBUG [RS:1;3456ee6a3164:36505 {}] zookeeper.ZKUtil(111): regionserver:36505-0x10047194fa70002, quorum=127.0.0.1:62345, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/3456ee6a3164,36505,1731736500481 2024-11-16T05:55:01,653 WARN [RS:1;3456ee6a3164:36505 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T05:55:01,654 INFO [RS:1;3456ee6a3164:36505 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T05:55:01,654 DEBUG [RS:1;3456ee6a3164:36505 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/WALs/3456ee6a3164,36505,1731736500481 2024-11-16T05:55:01,655 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [3456ee6a3164,33069,1731736500303] 2024-11-16T05:55:01,655 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [3456ee6a3164,36505,1731736500481] 2024-11-16T05:55:01,683 INFO [RS:1;3456ee6a3164:36505 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T05:55:01,683 INFO [RS:0;3456ee6a3164:33069 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T05:55:01,707 INFO [RS:0;3456ee6a3164:33069 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T05:55:01,707 INFO [RS:1;3456ee6a3164:36505 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T05:55:01,714 INFO [RS:0;3456ee6a3164:33069 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T05:55:01,715 INFO [RS:0;3456ee6a3164:33069 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:01,717 INFO [RS:1;3456ee6a3164:36505 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T05:55:01,718 INFO [RS:1;3456ee6a3164:36505 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:01,720 INFO [RS:1;3456ee6a3164:36505 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T05:55:01,720 INFO [RS:0;3456ee6a3164:33069 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T05:55:01,726 INFO [RS:1;3456ee6a3164:36505 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T05:55:01,726 INFO [RS:0;3456ee6a3164:33069 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T05:55:01,728 INFO [RS:1;3456ee6a3164:36505 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:01,728 INFO [RS:0;3456ee6a3164:33069 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:01,729 DEBUG [RS:0;3456ee6a3164:33069 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:01,729 DEBUG [RS:1;3456ee6a3164:36505 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:01,729 DEBUG [RS:0;3456ee6a3164:33069 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:01,729 DEBUG [RS:1;3456ee6a3164:36505 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:01,729 DEBUG [RS:0;3456ee6a3164:33069 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:01,729 DEBUG [RS:1;3456ee6a3164:36505 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:01,729 DEBUG [RS:0;3456ee6a3164:33069 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:01,729 DEBUG [RS:1;3456ee6a3164:36505 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:01,730 DEBUG [RS:0;3456ee6a3164:33069 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:01,730 DEBUG [RS:1;3456ee6a3164:36505 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:01,730 DEBUG [RS:0;3456ee6a3164:33069 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0, corePoolSize=2, maxPoolSize=2 2024-11-16T05:55:01,730 DEBUG [RS:1;3456ee6a3164:36505 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0, corePoolSize=2, maxPoolSize=2 2024-11-16T05:55:01,730 DEBUG [RS:0;3456ee6a3164:33069 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:01,730 DEBUG [RS:1;3456ee6a3164:36505 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:01,730 DEBUG [RS:0;3456ee6a3164:33069 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:01,730 DEBUG [RS:1;3456ee6a3164:36505 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:01,730 DEBUG [RS:0;3456ee6a3164:33069 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:01,730 DEBUG [RS:1;3456ee6a3164:36505 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:01,730 DEBUG [RS:0;3456ee6a3164:33069 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:01,731 DEBUG [RS:1;3456ee6a3164:36505 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:01,731 DEBUG [RS:0;3456ee6a3164:33069 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:01,731 DEBUG [RS:1;3456ee6a3164:36505 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:01,731 DEBUG [RS:0;3456ee6a3164:33069 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:01,731 DEBUG [RS:1;3456ee6a3164:36505 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:01,731 DEBUG [RS:0;3456ee6a3164:33069 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/3456ee6a3164:0, corePoolSize=3, maxPoolSize=3 2024-11-16T05:55:01,731 DEBUG [RS:1;3456ee6a3164:36505 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/3456ee6a3164:0, corePoolSize=3, maxPoolSize=3 2024-11-16T05:55:01,731 DEBUG [RS:0;3456ee6a3164:33069 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0, corePoolSize=3, maxPoolSize=3 2024-11-16T05:55:01,731 DEBUG [RS:1;3456ee6a3164:36505 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0, corePoolSize=3, maxPoolSize=3 2024-11-16T05:55:01,737 INFO [RS:0;3456ee6a3164:33069 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:01,738 INFO [RS:0;3456ee6a3164:33069 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:01,738 INFO [RS:0;3456ee6a3164:33069 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:01,738 INFO [RS:0;3456ee6a3164:33069 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:01,738 INFO [RS:0;3456ee6a3164:33069 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:01,738 INFO [RS:0;3456ee6a3164:33069 {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,33069,1731736500303-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T05:55:01,740 INFO [RS:1;3456ee6a3164:36505 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:01,740 INFO [RS:1;3456ee6a3164:36505 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:01,741 INFO [RS:1;3456ee6a3164:36505 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:01,741 INFO [RS:1;3456ee6a3164:36505 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:01,741 INFO [RS:1;3456ee6a3164:36505 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:01,741 INFO [RS:1;3456ee6a3164:36505 {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,36505,1731736500481-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T05:55:01,774 INFO [RS:0;3456ee6a3164:33069 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T05:55:01,775 INFO [RS:1;3456ee6a3164:36505 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T05:55:01,776 INFO [RS:0;3456ee6a3164:33069 {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,33069,1731736500303-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:01,776 INFO [RS:1;3456ee6a3164:36505 {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,36505,1731736500481-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:01,777 INFO [RS:0;3456ee6a3164:33069 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:01,777 INFO [RS:1;3456ee6a3164:36505 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:01,777 INFO [RS:1;3456ee6a3164:36505 {}] regionserver.Replication(171): 3456ee6a3164,36505,1731736500481 started 2024-11-16T05:55:01,777 INFO [RS:0;3456ee6a3164:33069 {}] regionserver.Replication(171): 3456ee6a3164,33069,1731736500303 started 2024-11-16T05:55:01,787 WARN [3456ee6a3164:38307 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-16T05:55:01,806 INFO [RS:1;3456ee6a3164:36505 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:01,806 INFO [RS:0;3456ee6a3164:33069 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:01,806 INFO [RS:0;3456ee6a3164:33069 {}] regionserver.HRegionServer(1482): Serving as 3456ee6a3164,33069,1731736500303, RpcServer on 3456ee6a3164/172.17.0.2:33069, sessionid=0x10047194fa70001 2024-11-16T05:55:01,806 INFO [RS:1;3456ee6a3164:36505 {}] regionserver.HRegionServer(1482): Serving as 3456ee6a3164,36505,1731736500481, RpcServer on 3456ee6a3164/172.17.0.2:36505, sessionid=0x10047194fa70002 2024-11-16T05:55:01,807 DEBUG [RS:1;3456ee6a3164:36505 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T05:55:01,807 DEBUG [RS:0;3456ee6a3164:33069 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T05:55:01,808 DEBUG [RS:1;3456ee6a3164:36505 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 3456ee6a3164,36505,1731736500481 2024-11-16T05:55:01,808 DEBUG [RS:0;3456ee6a3164:33069 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 3456ee6a3164,33069,1731736500303 2024-11-16T05:55:01,808 DEBUG [RS:1;3456ee6a3164:36505 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3456ee6a3164,36505,1731736500481' 2024-11-16T05:55:01,808 DEBUG [RS:0;3456ee6a3164:33069 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3456ee6a3164,33069,1731736500303' 2024-11-16T05:55:01,808 DEBUG [RS:1;3456ee6a3164:36505 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T05:55:01,808 DEBUG [RS:0;3456ee6a3164:33069 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T05:55:01,809 DEBUG [RS:0;3456ee6a3164:33069 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T05:55:01,809 DEBUG [RS:1;3456ee6a3164:36505 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T05:55:01,810 DEBUG [RS:1;3456ee6a3164:36505 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T05:55:01,810 DEBUG [RS:0;3456ee6a3164:33069 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T05:55:01,810 DEBUG [RS:1;3456ee6a3164:36505 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T05:55:01,810 DEBUG [RS:0;3456ee6a3164:33069 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T05:55:01,811 DEBUG [RS:1;3456ee6a3164:36505 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 3456ee6a3164,36505,1731736500481 2024-11-16T05:55:01,811 DEBUG [RS:0;3456ee6a3164:33069 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 3456ee6a3164,33069,1731736500303 2024-11-16T05:55:01,811 DEBUG [RS:1;3456ee6a3164:36505 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3456ee6a3164,36505,1731736500481' 2024-11-16T05:55:01,811 DEBUG [RS:0;3456ee6a3164:33069 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3456ee6a3164,33069,1731736500303' 2024-11-16T05:55:01,811 DEBUG [RS:1;3456ee6a3164:36505 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T05:55:01,811 DEBUG [RS:0;3456ee6a3164:33069 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T05:55:01,811 DEBUG [RS:1;3456ee6a3164:36505 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T05:55:01,811 DEBUG [RS:0;3456ee6a3164:33069 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T05:55:01,812 DEBUG [RS:0;3456ee6a3164:33069 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T05:55:01,812 DEBUG [RS:1;3456ee6a3164:36505 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T05:55:01,812 INFO [RS:1;3456ee6a3164:36505 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T05:55:01,812 INFO [RS:0;3456ee6a3164:33069 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T05:55:01,812 INFO [RS:0;3456ee6a3164:33069 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T05:55:01,812 INFO [RS:1;3456ee6a3164:36505 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T05:55:01,918 INFO [RS:1;3456ee6a3164:36505 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T05:55:01,918 INFO [RS:0;3456ee6a3164:33069 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T05:55:01,919 INFO [RS:0;3456ee6a3164:33069 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=3456ee6a3164%2C33069%2C1731736500303, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/WALs/3456ee6a3164,33069,1731736500303, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/oldWALs, maxLogs=55 2024-11-16T05:55:01,919 INFO [RS:1;3456ee6a3164:36505 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=3456ee6a3164%2C36505%2C1731736500481, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/WALs/3456ee6a3164,36505,1731736500481, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/oldWALs, maxLogs=55 2024-11-16T05:55:01,940 INFO [RS:0;3456ee6a3164:33069 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/WALs/3456ee6a3164,33069,1731736500303/3456ee6a3164%2C33069%2C1731736500303.1731736501923 2024-11-16T05:55:01,941 DEBUG [RS:0;3456ee6a3164:33069 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-16T05:55:01,946 INFO [RS:1;3456ee6a3164:36505 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/WALs/3456ee6a3164,36505,1731736500481/3456ee6a3164%2C36505%2C1731736500481.1731736501923 2024-11-16T05:55:01,947 DEBUG [RS:1;3456ee6a3164:36505 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-16T05:55:02,039 DEBUG [3456ee6a3164:38307 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-16T05:55:02,050 DEBUG [3456ee6a3164:38307 {}] balancer.BalancerClusterState(204): Hosts are {3456ee6a3164=0} racks are {/default-rack=0} 2024-11-16T05:55:02,053 DEBUG [3456ee6a3164:38307 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-16T05:55:02,053 DEBUG [3456ee6a3164:38307 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-16T05:55:02,053 DEBUG [3456ee6a3164:38307 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-16T05:55:02,053 DEBUG [3456ee6a3164:38307 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-16T05:55:02,053 INFO [3456ee6a3164:38307 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-16T05:55:02,054 INFO [3456ee6a3164:38307 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-16T05:55:02,054 DEBUG [3456ee6a3164:38307 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-16T05:55:02,062 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=3456ee6a3164,36505,1731736500481 2024-11-16T05:55:02,068 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3456ee6a3164,36505,1731736500481, state=OPENING 2024-11-16T05:55:02,073 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-16T05:55:02,076 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38307-0x10047194fa70000, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:55:02,076 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36505-0x10047194fa70002, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:55:02,076 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33069-0x10047194fa70001, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:55:02,077 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:55:02,077 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:55:02,077 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:55:02,079 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T05:55:02,080 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,36505,1731736500481}] 2024-11-16T05:55:02,252 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T05:55:02,256 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49599, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T05:55:02,264 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-16T05:55:02,265 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T05:55:02,266 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-16T05:55:02,267 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=3456ee6a3164%2C36505%2C1731736500481.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/WALs/3456ee6a3164,36505,1731736500481, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/oldWALs, maxLogs=55 2024-11-16T05:55:02,290 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/WALs/3456ee6a3164,36505,1731736500481/3456ee6a3164%2C36505%2C1731736500481.meta.1731736502269.meta 2024-11-16T05:55:02,291 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-16T05:55:02,292 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-16T05:55:02,294 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-16T05:55:02,297 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-16T05:55:02,302 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-16T05:55:02,306 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-16T05:55:02,306 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:55:02,307 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-16T05:55:02,307 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-16T05:55:02,308 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-16T05:55:02,309 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T05:55:02,309 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:55:02,310 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:55:02,310 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-16T05:55:02,311 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T05:55:02,311 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:55:02,311 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:55:02,311 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-16T05:55:02,312 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T05:55:02,312 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:55:02,312 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:55:02,312 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-16T05:55:02,313 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T05:55:02,313 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:55:02,313 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:55:02,313 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T05:55:02,314 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740 2024-11-16T05:55:02,315 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740 2024-11-16T05:55:02,316 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T05:55:02,316 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T05:55:02,316 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T05:55:02,317 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T05:55:02,318 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63999352, jitterRate=-0.04633533954620361}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T05:55:02,318 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-16T05:55:02,321 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731736502307Writing region info on filesystem at 1731736502308 (+1 ms)Initializing all the Stores at 1731736502308Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731736502308Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731736502308Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736502308Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731736502308Cleaning up temporary data from old regions at 1731736502316 (+8 ms)Running coprocessor post-open hooks at 1731736502318 (+2 ms)Region opened successfully at 1731736502320 (+2 ms) 2024-11-16T05:55:02,329 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731736502243 2024-11-16T05:55:02,345 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-16T05:55:02,345 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-16T05:55:02,346 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,36505,1731736500481 2024-11-16T05:55:02,348 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3456ee6a3164,36505,1731736500481, state=OPEN 2024-11-16T05:55:02,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38307-0x10047194fa70000, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:55:02,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33069-0x10047194fa70001, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:55:02,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36505-0x10047194fa70002, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:55:02,352 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:55:02,352 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:55:02,352 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:55:02,352 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=3456ee6a3164,36505,1731736500481 2024-11-16T05:55:02,356 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-16T05:55:02,356 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,36505,1731736500481 in 273 msec 2024-11-16T05:55:02,362 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-16T05:55:02,362 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 739 msec 2024-11-16T05:55:02,363 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T05:55:02,364 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-16T05:55:02,387 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:55:02,389 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,36505,1731736500481, seqNum=-1] 2024-11-16T05:55:02,413 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:55:02,415 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53985, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:55:02,436 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0660 sec 2024-11-16T05:55:02,436 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731736502436, completionTime=-1 2024-11-16T05:55:02,439 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-16T05:55:02,439 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T05:55:02,473 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-16T05:55:02,473 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731736562473 2024-11-16T05:55:02,474 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731736622474 2024-11-16T05:55:02,474 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 34 msec 2024-11-16T05:55:02,476 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-16T05:55:02,484 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,38307,1731736499374-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:02,484 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,38307,1731736499374-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:02,485 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,38307,1731736499374-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:02,487 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-3456ee6a3164:38307, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:02,487 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:02,488 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:02,503 DEBUG [master/3456ee6a3164:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-16T05:55:02,527 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.891sec 2024-11-16T05:55:02,530 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-16T05:55:02,532 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-16T05:55:02,533 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-16T05:55:02,534 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-16T05:55:02,534 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-16T05:55:02,535 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,38307,1731736499374-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T05:55:02,536 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,38307,1731736499374-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-16T05:55:02,537 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-16T05:55:02,538 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-16T05:55:02,539 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,38307,1731736499374-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:02,560 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e861955, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:55:02,563 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-16T05:55:02,563 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-16T05:55:02,568 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,38307,-1 for getting cluster id 2024-11-16T05:55:02,571 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:55:02,581 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4f9cc7e2-2fa1-4d4f-9f0f-9a4f2fa1dbe2' 2024-11-16T05:55:02,584 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:55:02,585 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4f9cc7e2-2fa1-4d4f-9f0f-9a4f2fa1dbe2" 2024-11-16T05:55:02,585 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@233fa0a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:55:02,585 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,38307,-1] 2024-11-16T05:55:02,589 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:55:02,591 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:55:02,594 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56134, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:55:02,597 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fbbc66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:55:02,598 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:55:02,608 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,36505,1731736500481, seqNum=-1] 2024-11-16T05:55:02,609 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:55:02,618 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49178, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:55:02,655 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=3456ee6a3164,38307,1731736499374 2024-11-16T05:55:02,666 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T05:55:02,672 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 3456ee6a3164,38307,1731736499374 2024-11-16T05:55:02,675 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@10fb0e3 2024-11-16T05:55:02,676 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T05:55:02,680 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56150, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T05:55:02,687 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38307 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1731736502655', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T05:55:02,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38307 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731736502655 2024-11-16T05:55:02,698 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731736502655 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T05:55:02,701 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38307 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1731736502655" procId is: 4 2024-11-16T05:55:02,701 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:55:02,703 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731736502655 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T05:55:02,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38307 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T05:55:02,719 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731736502655-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f3d530ff228e39408d8f0add39bda080, NAME => 'testBlockEvictionOnRegionMove1731736502655,,1731736502682.f3d530ff228e39408d8f0add39bda080.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1731736502655', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926 2024-11-16T05:55:02,726 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731736502655-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731736502655,,1731736502682.f3d530ff228e39408d8f0add39bda080.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:55:02,727 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731736502655-pool-0 {}] regionserver.HRegion(1722): Closing f3d530ff228e39408d8f0add39bda080, disabling compactions & flushes 2024-11-16T05:55:02,727 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731736502655-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731736502655,,1731736502682.f3d530ff228e39408d8f0add39bda080. 2024-11-16T05:55:02,727 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731736502655-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731736502655,,1731736502682.f3d530ff228e39408d8f0add39bda080. 2024-11-16T05:55:02,727 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731736502655-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731736502655,,1731736502682.f3d530ff228e39408d8f0add39bda080. after waiting 0 ms 2024-11-16T05:55:02,727 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731736502655-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731736502655,,1731736502682.f3d530ff228e39408d8f0add39bda080. 2024-11-16T05:55:02,727 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731736502655-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731736502655,,1731736502682.f3d530ff228e39408d8f0add39bda080. 2024-11-16T05:55:02,727 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731736502655-pool-0 {}] regionserver.HRegion(1676): Region close journal for f3d530ff228e39408d8f0add39bda080: Waiting for close lock at 1731736502727Disabling compacts and flushes for region at 1731736502727Disabling writes for close at 1731736502727Writing region close event to WAL at 1731736502727Closed at 1731736502727 2024-11-16T05:55:02,729 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731736502655 execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T05:55:02,733 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1731736502655,,1731736502682.f3d530ff228e39408d8f0add39bda080.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731736502729"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736502729"}]},"ts":"1731736502729"} 2024-11-16T05:55:02,738 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T05:55:02,739 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731736502655 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T05:55:02,741 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731736502655","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731736502739"}]},"ts":"1731736502739"} 2024-11-16T05:55:02,745 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731736502655, state=ENABLING in hbase:meta 2024-11-16T05:55:02,746 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {3456ee6a3164=0} racks are {/default-rack=0} 2024-11-16T05:55:02,746 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-16T05:55:02,747 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-16T05:55:02,747 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-16T05:55:02,747 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-16T05:55:02,747 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-16T05:55:02,747 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-16T05:55:02,747 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-16T05:55:02,748 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731736502655, region=f3d530ff228e39408d8f0add39bda080, ASSIGN}] 2024-11-16T05:55:02,750 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731736502655, region=f3d530ff228e39408d8f0add39bda080, ASSIGN 2024-11-16T05:55:02,751 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731736502655, region=f3d530ff228e39408d8f0add39bda080, ASSIGN; state=OFFLINE, location=3456ee6a3164,36505,1731736500481; forceNewPlan=false, retain=false 2024-11-16T05:55:02,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38307 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T05:55:02,904 INFO [3456ee6a3164:38307 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-16T05:55:02,905 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f3d530ff228e39408d8f0add39bda080, regionState=OPENING, regionLocation=3456ee6a3164,36505,1731736500481 2024-11-16T05:55:02,910 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731736502655, region=f3d530ff228e39408d8f0add39bda080, ASSIGN because future has completed 2024-11-16T05:55:02,911 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f3d530ff228e39408d8f0add39bda080, server=3456ee6a3164,36505,1731736500481}] 2024-11-16T05:55:03,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38307 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T05:55:03,068 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731736502655,,1731736502682.f3d530ff228e39408d8f0add39bda080. 2024-11-16T05:55:03,068 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => f3d530ff228e39408d8f0add39bda080, NAME => 'testBlockEvictionOnRegionMove1731736502655,,1731736502682.f3d530ff228e39408d8f0add39bda080.', STARTKEY => '', ENDKEY => ''} 2024-11-16T05:55:03,069 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731736502655 f3d530ff228e39408d8f0add39bda080 2024-11-16T05:55:03,069 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731736502655,,1731736502682.f3d530ff228e39408d8f0add39bda080.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:55:03,069 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for f3d530ff228e39408d8f0add39bda080 2024-11-16T05:55:03,070 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for f3d530ff228e39408d8f0add39bda080 2024-11-16T05:55:03,071 INFO [StoreOpener-f3d530ff228e39408d8f0add39bda080-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region f3d530ff228e39408d8f0add39bda080 2024-11-16T05:55:03,076 INFO [StoreOpener-f3d530ff228e39408d8f0add39bda080-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f3d530ff228e39408d8f0add39bda080 columnFamilyName family 2024-11-16T05:55:03,077 DEBUG [StoreOpener-f3d530ff228e39408d8f0add39bda080-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:55:03,078 INFO [StoreOpener-f3d530ff228e39408d8f0add39bda080-1 {}] regionserver.HStore(327): Store=f3d530ff228e39408d8f0add39bda080/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:55:03,078 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for f3d530ff228e39408d8f0add39bda080 2024-11-16T05:55:03,080 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/default/testBlockEvictionOnRegionMove1731736502655/f3d530ff228e39408d8f0add39bda080 2024-11-16T05:55:03,080 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/default/testBlockEvictionOnRegionMove1731736502655/f3d530ff228e39408d8f0add39bda080 2024-11-16T05:55:03,080 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for f3d530ff228e39408d8f0add39bda080 2024-11-16T05:55:03,080 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for f3d530ff228e39408d8f0add39bda080 2024-11-16T05:55:03,082 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for f3d530ff228e39408d8f0add39bda080 2024-11-16T05:55:03,090 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/default/testBlockEvictionOnRegionMove1731736502655/f3d530ff228e39408d8f0add39bda080/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:55:03,090 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened f3d530ff228e39408d8f0add39bda080; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72549660, jitterRate=0.08107417821884155}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T05:55:03,090 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f3d530ff228e39408d8f0add39bda080 2024-11-16T05:55:03,091 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for f3d530ff228e39408d8f0add39bda080: Running coprocessor pre-open hook at 1731736503070Writing region info on filesystem at 1731736503070Initializing all the Stores at 1731736503071 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736503071Cleaning up temporary data from old regions at 1731736503080 (+9 ms)Running coprocessor post-open hooks at 1731736503090 (+10 ms)Region opened successfully at 1731736503091 (+1 ms) 2024-11-16T05:55:03,092 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731736502655,,1731736502682.f3d530ff228e39408d8f0add39bda080., pid=6, masterSystemTime=1731736503065 2024-11-16T05:55:03,096 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731736502655,,1731736502682.f3d530ff228e39408d8f0add39bda080. 2024-11-16T05:55:03,096 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731736502655,,1731736502682.f3d530ff228e39408d8f0add39bda080. 2024-11-16T05:55:03,096 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f3d530ff228e39408d8f0add39bda080, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,36505,1731736500481 2024-11-16T05:55:03,100 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f3d530ff228e39408d8f0add39bda080, server=3456ee6a3164,36505,1731736500481 because future has completed 2024-11-16T05:55:03,103 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-16T05:55:03,104 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure f3d530ff228e39408d8f0add39bda080, server=3456ee6a3164,36505,1731736500481 in 190 msec 2024-11-16T05:55:03,107 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-16T05:55:03,107 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731736502655, region=f3d530ff228e39408d8f0add39bda080, ASSIGN in 355 msec 2024-11-16T05:55:03,108 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731736502655 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T05:55:03,108 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731736502655","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731736503108"}]},"ts":"1731736503108"} 2024-11-16T05:55:03,111 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731736502655, state=ENABLED in hbase:meta 2024-11-16T05:55:03,113 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731736502655 execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T05:55:03,116 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731736502655 in 422 msec 2024-11-16T05:55:03,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38307 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T05:55:03,334 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1731736502655 get assigned. Timeout = 60000ms 2024-11-16T05:55:03,334 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1731736502655 completed 2024-11-16T05:55:03,337 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T05:55:03,344 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1731736502655 assigned to meta. Checking AM states. 2024-11-16T05:55:03,345 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T05:55:03,346 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1731736502655 assigned. 2024-11-16T05:55:03,357 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1731736502655', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1731736502655,,1731736502682.f3d530ff228e39408d8f0add39bda080., hostname=3456ee6a3164,36505,1731736500481, seqNum=2] 2024-11-16T05:55:03,373 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f3d530ff228e39408d8f0add39bda080 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-16T05:55:03,408 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d79ad6a2582d43078a3251638bf59630 2024-11-16T05:55:03,460 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/default/testBlockEvictionOnRegionMove1731736502655/f3d530ff228e39408d8f0add39bda080/.tmp/family/d79ad6a2582d43078a3251638bf59630 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-16T05:55:03,467 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/default/testBlockEvictionOnRegionMove1731736502655/f3d530ff228e39408d8f0add39bda080/.tmp/family/d79ad6a2582d43078a3251638bf59630 2024-11-16T05:55:03,515 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/default/testBlockEvictionOnRegionMove1731736502655/f3d530ff228e39408d8f0add39bda080/.tmp/family/d79ad6a2582d43078a3251638bf59630 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/default/testBlockEvictionOnRegionMove1731736502655/f3d530ff228e39408d8f0add39bda080/family/d79ad6a2582d43078a3251638bf59630 2024-11-16T05:55:03,516 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/default/testBlockEvictionOnRegionMove1731736502655/f3d530ff228e39408d8f0add39bda080/family/d79ad6a2582d43078a3251638bf59630, delay=908 ms 2024-11-16T05:55:03,518 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/default/testBlockEvictionOnRegionMove1731736502655/f3d530ff228e39408d8f0add39bda080/family/d79ad6a2582d43078a3251638bf59630, entries=2, sequenceid=6, filesize=4.9 K 2024-11-16T05:55:03,524 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for f3d530ff228e39408d8f0add39bda080 in 150ms, sequenceid=6, compaction requested=false 2024-11-16T05:55:03,525 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1731736502655' 2024-11-16T05:55:03,527 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for f3d530ff228e39408d8f0add39bda080: 2024-11-16T05:55:04,427 DEBUG [hfile-prefetch-1731736503516 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d79ad6a2582d43078a3251638bf59630_0. 2024-11-16T05:55:04,431 DEBUG [hfile-prefetch-1731736503516 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d79ad6a2582d43078a3251638bf59630_117. 2024-11-16T05:55:04,432 DEBUG [hfile-prefetch-1731736503516 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/default/testBlockEvictionOnRegionMove1731736502655/f3d530ff228e39408d8f0add39bda080/family/d79ad6a2582d43078a3251638bf59630, with total blocks 2, and data blocks 1 2024-11-16T05:55:04,433 DEBUG [hfile-prefetch-1731736503516 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-16T05:55:04,433 DEBUG [hfile-prefetch-1731736503516 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/default/testBlockEvictionOnRegionMove1731736502655/f3d530ff228e39408d8f0add39bda080/family/d79ad6a2582d43078a3251638bf59630, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-16T05:55:04,433 DEBUG [hfile-prefetch-1731736503516 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/default/testBlockEvictionOnRegionMove1731736502655/f3d530ff228e39408d8f0add39bda080/family/d79ad6a2582d43078a3251638bf59630. Will sleep 100ms and try the verification again. 2024-11-16T05:55:04,509 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-16T05:55:04,509 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-16T05:55:04,509 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-16T05:55:04,509 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-16T05:55:04,516 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-16T05:55:04,516 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-16T05:55:04,516 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-16T05:55:04,516 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-16T05:55:04,533 ERROR [bucket-cache-persister {}] bucket.PersistentIOEngine(78): Calculating checksum failed, because of java.io.IOException: Stream closed at java.io.BufferedInputStream.getBufIfOpen(BufferedInputStream.java:168) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:334) ~[?:?] at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:281) ~[?:?] at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:324) ~[?:?] at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:189) ~[?:?] at java.io.InputStreamReader.read(InputStreamReader.java:177) ~[?:?] at java.io.BufferedReader.fill(BufferedReader.java:162) ~[?:?] at java.io.BufferedReader.read1(BufferedReader.java:213) ~[?:?] at java.io.BufferedReader.read(BufferedReader.java:287) ~[?:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.parseExecResult(Shell.java:1295) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.runCommand(Shell.java:1054) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.getFileSize(PersistentIOEngine.java:94) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.calculateChecksum(PersistentIOEngine.java:71) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.toPB(BucketProtoUtils.java:61) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.serializeAsPB(BucketProtoUtils.java:75) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistChunkedBackingMap(BucketCache.java:1678) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1496) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2024-11-16T05:55:04,534 DEBUG [hfile-prefetch-1731736503516 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/default/testBlockEvictionOnRegionMove1731736502655/f3d530ff228e39408d8f0add39bda080/family/d79ad6a2582d43078a3251638bf59630, with total blocks 2, and data blocks 1 2024-11-16T05:55:04,534 DEBUG [hfile-prefetch-1731736503516 {}] bucket.BucketCache(2285): iterating over 1 entries in the backing map 2024-11-16T05:55:04,534 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-16T05:55:04,534 DEBUG [hfile-prefetch-1731736503516 {}] bucket.BucketCache(2293): found block for file d79ad6a2582d43078a3251638bf59630 in the backing map. Acquiring read lock for offset 0 2024-11-16T05:55:04,534 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731736504509 2024-11-16T05:55:04,534 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-16T05:55:04,534 DEBUG [hfile-prefetch-1731736503516 {}] bucket.BucketCache(2305): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/default/testBlockEvictionOnRegionMove1731736502655/f3d530ff228e39408d8f0add39bda080/family/d79ad6a2582d43078a3251638bf59630 has now been fully cached. 2024-11-16T05:55:04,534 WARN [Thread-88 {}] util.Shell$1(1036): Error reading the error stream java.io.IOException: Stream closed at java.io.BufferedReader.ensureOpen(BufferedReader.java:123) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:321) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:396) ~[?:?] at org.apache.hadoop.util.Shell$1.run(Shell.java:1025) ~[hadoop-common-3.4.1.jar:?] 2024-11-16T05:55:04,534 DEBUG [hfile-prefetch-1731736503516 {}] hfile.PrefetchExecutor(121): Prefetch completed for d79ad6a2582d43078a3251638bf59630 2024-11-16T05:55:04,540 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-16T05:55:04,540 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1731736504516 2024-11-16T05:55:04,540 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-16T05:55:04,543 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38307 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=f3d530ff228e39408d8f0add39bda080, source=3456ee6a3164,36505,1731736500481, destination=3456ee6a3164,33069,1731736500303, warming up region on 3456ee6a3164,33069,1731736500303 2024-11-16T05:55:04,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38307 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T05:55:04,547 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38307 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=f3d530ff228e39408d8f0add39bda080, source=3456ee6a3164,36505,1731736500481, destination=3456ee6a3164,33069,1731736500303, running balancer 2024-11-16T05:55:04,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38307 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731736502655, region=f3d530ff228e39408d8f0add39bda080, REOPEN/MOVE 2024-11-16T05:55:04,549 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731736502655, region=f3d530ff228e39408d8f0add39bda080, REOPEN/MOVE 2024-11-16T05:55:04,549 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59551, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T05:55:04,551 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33069 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1731736502655,,1731736502682.f3d530ff228e39408d8f0add39bda080. 2024-11-16T05:55:04,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33069 {}] regionserver.HRegion(7855): Warmup {ENCODED => f3d530ff228e39408d8f0add39bda080, NAME => 'testBlockEvictionOnRegionMove1731736502655,,1731736502682.f3d530ff228e39408d8f0add39bda080.', STARTKEY => '', ENDKEY => ''} 2024-11-16T05:55:04,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33069 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731736502655,,1731736502682.f3d530ff228e39408d8f0add39bda080.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:55:04,552 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=f3d530ff228e39408d8f0add39bda080, regionState=CLOSING, regionLocation=3456ee6a3164,36505,1731736500481 2024-11-16T05:55:04,554 INFO [StoreOpener-f3d530ff228e39408d8f0add39bda080-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region f3d530ff228e39408d8f0add39bda080 2024-11-16T05:55:04,555 INFO [StoreOpener-f3d530ff228e39408d8f0add39bda080-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f3d530ff228e39408d8f0add39bda080 columnFamilyName family 2024-11-16T05:55:04,555 DEBUG [StoreOpener-f3d530ff228e39408d8f0add39bda080-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:55:04,556 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731736502655, region=f3d530ff228e39408d8f0add39bda080, REOPEN/MOVE because future has completed 2024-11-16T05:55:04,561 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-16T05:55:04,562 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure f3d530ff228e39408d8f0add39bda080, server=3456ee6a3164,36505,1731736500481}] 2024-11-16T05:55:04,564 DEBUG [StoreOpener-f3d530ff228e39408d8f0add39bda080-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/default/testBlockEvictionOnRegionMove1731736502655/f3d530ff228e39408d8f0add39bda080/family/d79ad6a2582d43078a3251638bf59630 2024-11-16T05:55:04,564 INFO [StoreOpener-f3d530ff228e39408d8f0add39bda080-1 {}] regionserver.HStore(327): Store=f3d530ff228e39408d8f0add39bda080/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:55:04,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33069 {}] regionserver.HRegion(1722): Closing f3d530ff228e39408d8f0add39bda080, disabling compactions & flushes 2024-11-16T05:55:04,565 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33069 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731736502655,,1731736502682.f3d530ff228e39408d8f0add39bda080. 2024-11-16T05:55:04,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33069 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731736502655,,1731736502682.f3d530ff228e39408d8f0add39bda080. 2024-11-16T05:55:04,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33069 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731736502655,,1731736502682.f3d530ff228e39408d8f0add39bda080. after waiting 0 ms 2024-11-16T05:55:04,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33069 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731736502655,,1731736502682.f3d530ff228e39408d8f0add39bda080. 2024-11-16T05:55:04,567 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33069 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731736502655,,1731736502682.f3d530ff228e39408d8f0add39bda080. 2024-11-16T05:55:04,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33069 {}] regionserver.HRegion(1676): Region close journal for f3d530ff228e39408d8f0add39bda080: Waiting for close lock at 1731736504565Disabling compacts and flushes for region at 1731736504565Disabling writes for close at 1731736504565Writing region close event to WAL at 1731736504567 (+2 ms)Closed at 1731736504567 2024-11-16T05:55:04,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38307 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-16T05:55:04,719 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close f3d530ff228e39408d8f0add39bda080 2024-11-16T05:55:04,719 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-16T05:55:04,720 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing f3d530ff228e39408d8f0add39bda080, disabling compactions & flushes 2024-11-16T05:55:04,720 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731736502655,,1731736502682.f3d530ff228e39408d8f0add39bda080. 2024-11-16T05:55:04,720 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731736502655,,1731736502682.f3d530ff228e39408d8f0add39bda080. 2024-11-16T05:55:04,720 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731736502655,,1731736502682.f3d530ff228e39408d8f0add39bda080. after waiting 0 ms 2024-11-16T05:55:04,720 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731736502655,,1731736502682.f3d530ff228e39408d8f0add39bda080. 2024-11-16T05:55:04,722 DEBUG [StoreFileCloser-f3d530ff228e39408d8f0add39bda080-family-1 {}] bucket.BucketCache(768): Removing file d79ad6a2582d43078a3251638bf59630 for region f3d530ff228e39408d8f0add39bda080 2024-11-16T05:55:04,723 DEBUG [StoreFileCloser-f3d530ff228e39408d8f0add39bda080-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file d79ad6a2582d43078a3251638bf59630, starting offset: 0, end offset: 9223372036854775807 2024-11-16T05:55:04,724 DEBUG [StoreFileCloser-f3d530ff228e39408d8f0add39bda080-family-1 {}] bucket.BucketCache(856): removed key d79ad6a2582d43078a3251638bf59630_0 from back map with offset lock 2094080 in the evict process 2024-11-16T05:55:04,733 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/default/testBlockEvictionOnRegionMove1731736502655/f3d530ff228e39408d8f0add39bda080/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-16T05:55:04,734 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-16T05:55:04,734 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-16T05:55:04,734 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-16T05:55:04,735 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-16T05:55:04,735 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731736502655,,1731736502682.f3d530ff228e39408d8f0add39bda080. 2024-11-16T05:55:04,735 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for f3d530ff228e39408d8f0add39bda080: Waiting for close lock at 1731736504720Running coprocessor pre-close hooks at 1731736504720Disabling compacts and flushes for region at 1731736504720Disabling writes for close at 1731736504720Writing region close event to WAL at 1731736504724 (+4 ms)Running coprocessor post-close hooks at 1731736504734 (+10 ms)Closed at 1731736504735 (+1 ms) 2024-11-16T05:55:04,736 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding f3d530ff228e39408d8f0add39bda080 move to 3456ee6a3164,33069,1731736500303 record at close sequenceid=6 2024-11-16T05:55:04,738 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-16T05:55:04,738 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731736504734 2024-11-16T05:55:04,738 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-16T05:55:04,739 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed f3d530ff228e39408d8f0add39bda080 2024-11-16T05:55:04,740 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=f3d530ff228e39408d8f0add39bda080, regionState=CLOSED 2024-11-16T05:55:04,744 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure f3d530ff228e39408d8f0add39bda080, server=3456ee6a3164,36505,1731736500481 because future has completed 2024-11-16T05:55:04,747 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-16T05:55:04,747 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure f3d530ff228e39408d8f0add39bda080, server=3456ee6a3164,36505,1731736500481 in 183 msec 2024-11-16T05:55:04,748 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731736502655, region=f3d530ff228e39408d8f0add39bda080, REOPEN/MOVE; state=CLOSED, location=3456ee6a3164,33069,1731736500303; forceNewPlan=false, retain=false 2024-11-16T05:55:04,898 INFO [3456ee6a3164:38307 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-16T05:55:04,899 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=f3d530ff228e39408d8f0add39bda080, regionState=OPENING, regionLocation=3456ee6a3164,33069,1731736500303 2024-11-16T05:55:04,902 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731736502655, region=f3d530ff228e39408d8f0add39bda080, REOPEN/MOVE because future has completed 2024-11-16T05:55:04,903 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure f3d530ff228e39408d8f0add39bda080, server=3456ee6a3164,33069,1731736500303}] 2024-11-16T05:55:05,058 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731736502655,,1731736502682.f3d530ff228e39408d8f0add39bda080. 2024-11-16T05:55:05,058 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => f3d530ff228e39408d8f0add39bda080, NAME => 'testBlockEvictionOnRegionMove1731736502655,,1731736502682.f3d530ff228e39408d8f0add39bda080.', STARTKEY => '', ENDKEY => ''} 2024-11-16T05:55:05,059 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731736502655 f3d530ff228e39408d8f0add39bda080 2024-11-16T05:55:05,059 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731736502655,,1731736502682.f3d530ff228e39408d8f0add39bda080.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:55:05,059 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for f3d530ff228e39408d8f0add39bda080 2024-11-16T05:55:05,059 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for f3d530ff228e39408d8f0add39bda080 2024-11-16T05:55:05,061 INFO [StoreOpener-f3d530ff228e39408d8f0add39bda080-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region f3d530ff228e39408d8f0add39bda080 2024-11-16T05:55:05,062 INFO [StoreOpener-f3d530ff228e39408d8f0add39bda080-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f3d530ff228e39408d8f0add39bda080 columnFamilyName family 2024-11-16T05:55:05,062 DEBUG [StoreOpener-f3d530ff228e39408d8f0add39bda080-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:55:05,064 DEBUG [StoreFileOpener-f3d530ff228e39408d8f0add39bda080-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/default/testBlockEvictionOnRegionMove1731736502655/f3d530ff228e39408d8f0add39bda080/family/d79ad6a2582d43078a3251638bf59630, delay=995 ms 2024-11-16T05:55:05,065 DEBUG [StoreOpener-f3d530ff228e39408d8f0add39bda080-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/default/testBlockEvictionOnRegionMove1731736502655/f3d530ff228e39408d8f0add39bda080/family/d79ad6a2582d43078a3251638bf59630 2024-11-16T05:55:05,066 INFO [StoreOpener-f3d530ff228e39408d8f0add39bda080-1 {}] regionserver.HStore(327): Store=f3d530ff228e39408d8f0add39bda080/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:55:05,066 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for f3d530ff228e39408d8f0add39bda080 2024-11-16T05:55:05,067 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/default/testBlockEvictionOnRegionMove1731736502655/f3d530ff228e39408d8f0add39bda080 2024-11-16T05:55:05,067 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/default/testBlockEvictionOnRegionMove1731736502655/f3d530ff228e39408d8f0add39bda080 2024-11-16T05:55:05,067 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for f3d530ff228e39408d8f0add39bda080 2024-11-16T05:55:05,067 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for f3d530ff228e39408d8f0add39bda080 2024-11-16T05:55:05,071 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for f3d530ff228e39408d8f0add39bda080 2024-11-16T05:55:05,072 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened f3d530ff228e39408d8f0add39bda080; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61928955, jitterRate=-0.07718665897846222}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T05:55:05,072 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f3d530ff228e39408d8f0add39bda080 2024-11-16T05:55:05,073 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for f3d530ff228e39408d8f0add39bda080: Running coprocessor pre-open hook at 1731736505060Writing region info on filesystem at 1731736505060Initializing all the Stores at 1731736505060Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736505060Cleaning up temporary data from old regions at 1731736505067 (+7 ms)Running coprocessor post-open hooks at 1731736505072 (+5 ms)Region opened successfully at 1731736505073 (+1 ms) 2024-11-16T05:55:05,074 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731736502655,,1731736502682.f3d530ff228e39408d8f0add39bda080., pid=9, masterSystemTime=1731736505055 2024-11-16T05:55:05,077 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731736502655,,1731736502682.f3d530ff228e39408d8f0add39bda080. 2024-11-16T05:55:05,077 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731736502655,,1731736502682.f3d530ff228e39408d8f0add39bda080. 2024-11-16T05:55:05,077 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=f3d530ff228e39408d8f0add39bda080, regionState=OPEN, openSeqNum=10, regionLocation=3456ee6a3164,33069,1731736500303 2024-11-16T05:55:05,080 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure f3d530ff228e39408d8f0add39bda080, server=3456ee6a3164,33069,1731736500303 because future has completed 2024-11-16T05:55:05,083 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-16T05:55:05,083 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure f3d530ff228e39408d8f0add39bda080, server=3456ee6a3164,33069,1731736500303 in 178 msec 2024-11-16T05:55:05,085 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731736502655, region=f3d530ff228e39408d8f0add39bda080, REOPEN/MOVE in 536 msec 2024-11-16T05:55:05,104 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-16T05:55:05,105 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T05:55:05,105 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T05:55:05,110 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:55:05,110 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:55:05,110 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:55:05,111 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-16T05:55:05,111 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1596482003, stopped=false 2024-11-16T05:55:05,111 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=3456ee6a3164,38307,1731736499374 2024-11-16T05:55:05,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36505-0x10047194fa70002, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T05:55:05,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33069-0x10047194fa70001, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T05:55:05,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38307-0x10047194fa70000, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T05:55:05,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38307-0x10047194fa70000, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:55:05,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33069-0x10047194fa70001, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:55:05,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36505-0x10047194fa70002, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:55:05,113 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:55:05,114 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33069-0x10047194fa70001, quorum=127.0.0.1:62345, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T05:55:05,114 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38307-0x10047194fa70000, quorum=127.0.0.1:62345, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T05:55:05,114 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T05:55:05,114 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36505-0x10047194fa70002, quorum=127.0.0.1:62345, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T05:55:05,114 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T05:55:05,114 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:55:05,115 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '3456ee6a3164,33069,1731736500303' ***** 2024-11-16T05:55:05,115 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-16T05:55:05,115 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '3456ee6a3164,36505,1731736500481' ***** 2024-11-16T05:55:05,115 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-16T05:55:05,115 INFO [RS:0;3456ee6a3164:33069 {}] bucket.BucketCache(1738): Disabling cache 2024-11-16T05:55:05,116 INFO [RS:1;3456ee6a3164:36505 {}] bucket.BucketCache(1738): Disabling cache 2024-11-16T05:55:05,116 INFO [RS:1;3456ee6a3164:36505 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/bucket.persistence 2024-11-16T05:55:05,116 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-16T05:55:05,116 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-16T05:55:05,116 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-16T05:55:05,116 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-16T05:55:05,116 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-16T05:55:05,116 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-16T05:55:05,116 INFO [RS:1;3456ee6a3164:36505 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-16T05:55:05,116 INFO [RS:0;3456ee6a3164:33069 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/bucket.persistence 2024-11-16T05:55:05,116 INFO [RS:0;3456ee6a3164:33069 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-16T05:55:05,116 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-16T05:55:05,116 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-16T05:55:05,126 DEBUG [RS:0;3456ee6a3164:33069 {}] bucket.BucketCache(1487): Thread RS:0;3456ee6a3164:33069 started persisting bucket cache to file 2024-11-16T05:55:05,126 DEBUG [RS:1;3456ee6a3164:36505 {}] bucket.BucketCache(1487): Thread RS:1;3456ee6a3164:36505 started persisting bucket cache to file 2024-11-16T05:55:05,127 DEBUG [RS:1;3456ee6a3164:36505 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-16T05:55:05,127 DEBUG [RS:0;3456ee6a3164:33069 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-16T05:55:05,127 DEBUG [RS:1;3456ee6a3164:36505 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-16T05:55:05,127 DEBUG [RS:0;3456ee6a3164:33069 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-16T05:55:05,130 DEBUG [RS:1;3456ee6a3164:36505 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-16T05:55:05,130 DEBUG [RS:0;3456ee6a3164:33069 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-16T05:55:05,131 DEBUG [RS:0;3456ee6a3164:33069 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731736505127 2024-11-16T05:55:05,131 DEBUG [RS:1;3456ee6a3164:36505 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731736505127 2024-11-16T05:55:05,131 DEBUG [RS:1;3456ee6a3164:36505 {}] bucket.BucketCache(1509): Thread RS:1;3456ee6a3164:36505 finished persisting bucket cache to file, renaming 2024-11-16T05:55:05,131 DEBUG [RS:0;3456ee6a3164:33069 {}] bucket.BucketCache(1509): Thread RS:0;3456ee6a3164:33069 finished persisting bucket cache to file, renaming 2024-11-16T05:55:05,131 WARN [RS:0;3456ee6a3164:33069 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-16T05:55:05,131 INFO [RS:0;3456ee6a3164:33069 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T05:55:05,131 INFO [RS:1;3456ee6a3164:36505 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T05:55:05,131 INFO [RS:0;3456ee6a3164:33069 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-16T05:55:05,131 INFO [RS:1;3456ee6a3164:36505 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-16T05:55:05,131 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T05:55:05,131 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T05:55:05,131 INFO [RS:1;3456ee6a3164:36505 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-16T05:55:05,131 INFO [RS:0;3456ee6a3164:33069 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-16T05:55:05,132 INFO [RS:1;3456ee6a3164:36505 {}] regionserver.HRegionServer(959): stopping server 3456ee6a3164,36505,1731736500481 2024-11-16T05:55:05,132 INFO [RS:0;3456ee6a3164:33069 {}] regionserver.HRegionServer(3091): Received CLOSE for f3d530ff228e39408d8f0add39bda080 2024-11-16T05:55:05,132 INFO [RS:1;3456ee6a3164:36505 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T05:55:05,132 INFO [RS:1;3456ee6a3164:36505 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;3456ee6a3164:36505. 2024-11-16T05:55:05,132 DEBUG [RS:1;3456ee6a3164:36505 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T05:55:05,132 DEBUG [RS:1;3456ee6a3164:36505 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:55:05,132 INFO [RS:1;3456ee6a3164:36505 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-16T05:55:05,132 INFO [RS:1;3456ee6a3164:36505 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-16T05:55:05,132 INFO [RS:1;3456ee6a3164:36505 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-16T05:55:05,132 INFO [RS:1;3456ee6a3164:36505 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-16T05:55:05,133 INFO [RS:0;3456ee6a3164:33069 {}] regionserver.HRegionServer(959): stopping server 3456ee6a3164,33069,1731736500303 2024-11-16T05:55:05,133 INFO [RS:0;3456ee6a3164:33069 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T05:55:05,133 INFO [RS:0;3456ee6a3164:33069 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;3456ee6a3164:33069. 2024-11-16T05:55:05,133 DEBUG [RS:0;3456ee6a3164:33069 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T05:55:05,133 DEBUG [RS:0;3456ee6a3164:33069 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:55:05,133 INFO [RS:1;3456ee6a3164:36505 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-16T05:55:05,133 INFO [RS:0;3456ee6a3164:33069 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-16T05:55:05,133 DEBUG [RS:1;3456ee6a3164:36505 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-16T05:55:05,133 DEBUG [RS:0;3456ee6a3164:33069 {}] regionserver.HRegionServer(1325): Online Regions={f3d530ff228e39408d8f0add39bda080=testBlockEvictionOnRegionMove1731736502655,,1731736502682.f3d530ff228e39408d8f0add39bda080.} 2024-11-16T05:55:05,133 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1731736502655,,1731736502682.f3d530ff228e39408d8f0add39bda080. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-16T05:55:05,134 DEBUG [RS:0;3456ee6a3164:33069 {}] regionserver.HRegionServer(1351): Waiting on f3d530ff228e39408d8f0add39bda080 2024-11-16T05:55:05,133 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f3d530ff228e39408d8f0add39bda080, disabling compactions & flushes 2024-11-16T05:55:05,134 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-16T05:55:05,134 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731736502655,,1731736502682.f3d530ff228e39408d8f0add39bda080. 2024-11-16T05:55:05,134 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731736502655,,1731736502682.f3d530ff228e39408d8f0add39bda080. 2024-11-16T05:55:05,134 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T05:55:05,134 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731736502655,,1731736502682.f3d530ff228e39408d8f0add39bda080. after waiting 0 ms 2024-11-16T05:55:05,134 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T05:55:05,134 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731736502655,,1731736502682.f3d530ff228e39408d8f0add39bda080. 2024-11-16T05:55:05,134 DEBUG [RS:1;3456ee6a3164:36505 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T05:55:05,134 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T05:55:05,134 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T05:55:05,134 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T05:55:05,134 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-16T05:55:05,134 DEBUG [StoreFileCloser-f3d530ff228e39408d8f0add39bda080-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/default/testBlockEvictionOnRegionMove1731736502655/f3d530ff228e39408d8f0add39bda080/family/d79ad6a2582d43078a3251638bf59630 2024-11-16T05:55:05,139 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/default/testBlockEvictionOnRegionMove1731736502655/f3d530ff228e39408d8f0add39bda080/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-16T05:55:05,140 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731736502655,,1731736502682.f3d530ff228e39408d8f0add39bda080. 2024-11-16T05:55:05,140 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f3d530ff228e39408d8f0add39bda080: Waiting for close lock at 1731736505133Running coprocessor pre-close hooks at 1731736505133Disabling compacts and flushes for region at 1731736505133Disabling writes for close at 1731736505134 (+1 ms)Writing region close event to WAL at 1731736505135 (+1 ms)Running coprocessor post-close hooks at 1731736505139 (+4 ms)Closed at 1731736505139 2024-11-16T05:55:05,140 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1731736502655,,1731736502682.f3d530ff228e39408d8f0add39bda080. 2024-11-16T05:55:05,145 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a4da8c53c78a4746881d84e29af46b8b 2024-11-16T05:55:05,148 INFO [regionserver/3456ee6a3164:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T05:55:05,152 INFO [regionserver/3456ee6a3164:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T05:55:05,168 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/.tmp/info/a4da8c53c78a4746881d84e29af46b8b is 205, key is testBlockEvictionOnRegionMove1731736502655,,1731736502682.f3d530ff228e39408d8f0add39bda080./info:regioninfo/1731736505077/Put/seqid=0 2024-11-16T05:55:05,170 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/.tmp/info/a4da8c53c78a4746881d84e29af46b8b 2024-11-16T05:55:05,180 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6ead2e6ea52e430998612e8a03b39355 2024-11-16T05:55:05,199 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/.tmp/ns/6ead2e6ea52e430998612e8a03b39355 is 43, key is default/ns:d/1731736502419/Put/seqid=0 2024-11-16T05:55:05,200 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/.tmp/ns/6ead2e6ea52e430998612e8a03b39355 2024-11-16T05:55:05,215 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8d0794ef9e7446b3850e6237d7b291be 2024-11-16T05:55:05,233 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/.tmp/table/8d0794ef9e7446b3850e6237d7b291be is 78, key is testBlockEvictionOnRegionMove1731736502655/table:state/1731736503108/Put/seqid=0 2024-11-16T05:55:05,234 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/.tmp/table/8d0794ef9e7446b3850e6237d7b291be 2024-11-16T05:55:05,237 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/.tmp/info/a4da8c53c78a4746881d84e29af46b8b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/info/a4da8c53c78a4746881d84e29af46b8b 2024-11-16T05:55:05,239 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/info/a4da8c53c78a4746881d84e29af46b8b, delay=946 ms 2024-11-16T05:55:05,241 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/info/a4da8c53c78a4746881d84e29af46b8b, entries=15, sequenceid=15, filesize=7.6 K 2024-11-16T05:55:05,242 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/.tmp/ns/6ead2e6ea52e430998612e8a03b39355 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/ns/6ead2e6ea52e430998612e8a03b39355 2024-11-16T05:55:05,244 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/ns/6ead2e6ea52e430998612e8a03b39355, delay=933 ms 2024-11-16T05:55:05,247 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/ns/6ead2e6ea52e430998612e8a03b39355, entries=2, sequenceid=15, filesize=5.0 K 2024-11-16T05:55:05,247 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/.tmp/table/8d0794ef9e7446b3850e6237d7b291be as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/table/8d0794ef9e7446b3850e6237d7b291be 2024-11-16T05:55:05,249 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/table/8d0794ef9e7446b3850e6237d7b291be, delay=939 ms 2024-11-16T05:55:05,251 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/table/8d0794ef9e7446b3850e6237d7b291be, entries=2, sequenceid=15, filesize=5.3 K 2024-11-16T05:55:05,253 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 118ms, sequenceid=15, compaction requested=false 2024-11-16T05:55:05,253 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-16T05:55:05,256 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/info/a4da8c53c78a4746881d84e29af46b8b 2024-11-16T05:55:05,259 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/ns/6ead2e6ea52e430998612e8a03b39355 2024-11-16T05:55:05,261 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/table/8d0794ef9e7446b3850e6237d7b291be 2024-11-16T05:55:05,267 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-16T05:55:05,267 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-16T05:55:05,268 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T05:55:05,268 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731736505133Running coprocessor pre-close hooks at 1731736505134 (+1 ms)Disabling compacts and flushes for region at 1731736505134Disabling writes for close at 1731736505134Obtaining lock to block concurrent updates at 1731736505134Preparing flush snapshotting stores in 1588230740 at 1731736505134Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1731736505135 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731736505135Flushing 1588230740/info: creating writer at 1731736505136 (+1 ms)Flushing 1588230740/info: appending metadata at 1731736505167 (+31 ms)Flushing 1588230740/info: closing flushed file at 1731736505167Flushing 1588230740/ns: creating writer at 1731736505174 (+7 ms)Flushing 1588230740/ns: appending metadata at 1731736505198 (+24 ms)Flushing 1588230740/ns: closing flushed file at 1731736505198Flushing 1588230740/table: creating writer at 1731736505204 (+6 ms)Flushing 1588230740/table: appending metadata at 1731736505233 (+29 ms)Flushing 1588230740/table: closing flushed file at 1731736505233Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a8a7526: reopening flushed file at 1731736505237 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2158090: reopening flushed file at 1731736505242 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2521e1b4: reopening flushed file at 1731736505247 (+5 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 118ms, sequenceid=15, compaction requested=false at 1731736505253 (+6 ms)Writing region close event to WAL at 1731736505261 (+8 ms)Running coprocessor post-close hooks at 1731736505267 (+6 ms)Closed at 1731736505268 (+1 ms) 2024-11-16T05:55:05,268 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-16T05:55:05,334 INFO [RS:0;3456ee6a3164:33069 {}] regionserver.HRegionServer(976): stopping server 3456ee6a3164,33069,1731736500303; all regions closed. 2024-11-16T05:55:05,334 INFO [RS:1;3456ee6a3164:36505 {}] regionserver.HRegionServer(976): stopping server 3456ee6a3164,36505,1731736500481; all regions closed. 2024-11-16T05:55:05,344 DEBUG [RS:0;3456ee6a3164:33069 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/oldWALs 2024-11-16T05:55:05,344 DEBUG [RS:1;3456ee6a3164:36505 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/oldWALs 2024-11-16T05:55:05,344 INFO [RS:0;3456ee6a3164:33069 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3456ee6a3164%2C33069%2C1731736500303:(num 1731736501923) 2024-11-16T05:55:05,344 INFO [RS:1;3456ee6a3164:36505 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3456ee6a3164%2C36505%2C1731736500481.meta:.meta(num 1731736502269) 2024-11-16T05:55:05,344 DEBUG [RS:0;3456ee6a3164:33069 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:55:05,344 INFO [RS:0;3456ee6a3164:33069 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T05:55:05,344 INFO [RS:0;3456ee6a3164:33069 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T05:55:05,345 INFO [RS:0;3456ee6a3164:33069 {}] hbase.ChoreService(370): Chore service for: regionserver/3456ee6a3164:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-16T05:55:05,345 INFO [RS:0;3456ee6a3164:33069 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-16T05:55:05,345 INFO [regionserver/3456ee6a3164:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T05:55:05,345 INFO [RS:0;3456ee6a3164:33069 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-16T05:55:05,345 INFO [RS:0;3456ee6a3164:33069 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-16T05:55:05,345 INFO [RS:0;3456ee6a3164:33069 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T05:55:05,346 INFO [RS:0;3456ee6a3164:33069 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33069 2024-11-16T05:55:05,347 DEBUG [RS:1;3456ee6a3164:36505 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/oldWALs 2024-11-16T05:55:05,347 INFO [RS:1;3456ee6a3164:36505 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3456ee6a3164%2C36505%2C1731736500481:(num 1731736501923) 2024-11-16T05:55:05,347 DEBUG [RS:1;3456ee6a3164:36505 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:55:05,347 INFO [RS:1;3456ee6a3164:36505 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T05:55:05,347 INFO [RS:1;3456ee6a3164:36505 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T05:55:05,347 INFO [RS:1;3456ee6a3164:36505 {}] hbase.ChoreService(370): Chore service for: regionserver/3456ee6a3164:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-16T05:55:05,348 INFO [regionserver/3456ee6a3164:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T05:55:05,348 INFO [RS:1;3456ee6a3164:36505 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T05:55:05,349 INFO [RS:1;3456ee6a3164:36505 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36505 2024-11-16T05:55:05,352 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33069-0x10047194fa70001, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/3456ee6a3164,33069,1731736500303 2024-11-16T05:55:05,352 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38307-0x10047194fa70000, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T05:55:05,352 INFO [RS:0;3456ee6a3164:33069 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T05:55:05,354 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36505-0x10047194fa70002, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/3456ee6a3164,36505,1731736500481 2024-11-16T05:55:05,354 INFO [RS:1;3456ee6a3164:36505 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T05:55:05,356 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [3456ee6a3164,33069,1731736500303] 2024-11-16T05:55:05,363 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/3456ee6a3164,33069,1731736500303 already deleted, retry=false 2024-11-16T05:55:05,363 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 3456ee6a3164,33069,1731736500303 expired; onlineServers=1 2024-11-16T05:55:05,363 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [3456ee6a3164,36505,1731736500481] 2024-11-16T05:55:05,365 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/3456ee6a3164,36505,1731736500481 already deleted, retry=false 2024-11-16T05:55:05,365 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 3456ee6a3164,36505,1731736500481 expired; onlineServers=0 2024-11-16T05:55:05,365 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '3456ee6a3164,38307,1731736499374' ***** 2024-11-16T05:55:05,365 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-16T05:55:05,365 INFO [M:0;3456ee6a3164:38307 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T05:55:05,365 INFO [M:0;3456ee6a3164:38307 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T05:55:05,366 DEBUG [M:0;3456ee6a3164:38307 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-16T05:55:05,366 DEBUG [M:0;3456ee6a3164:38307 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-16T05:55:05,366 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-16T05:55:05,366 DEBUG [master/3456ee6a3164:0:becomeActiveMaster-HFileCleaner.large.0-1731736501517 {}] cleaner.HFileCleaner(306): Exit Thread[master/3456ee6a3164:0:becomeActiveMaster-HFileCleaner.large.0-1731736501517,5,FailOnTimeoutGroup] 2024-11-16T05:55:05,366 INFO [M:0;3456ee6a3164:38307 {}] hbase.ChoreService(370): Chore service for: master/3456ee6a3164:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-16T05:55:05,366 INFO [M:0;3456ee6a3164:38307 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T05:55:05,366 DEBUG [M:0;3456ee6a3164:38307 {}] master.HMaster(1795): Stopping service threads 2024-11-16T05:55:05,366 INFO [M:0;3456ee6a3164:38307 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-16T05:55:05,366 DEBUG [master/3456ee6a3164:0:becomeActiveMaster-HFileCleaner.small.0-1731736501526 {}] cleaner.HFileCleaner(306): Exit Thread[master/3456ee6a3164:0:becomeActiveMaster-HFileCleaner.small.0-1731736501526,5,FailOnTimeoutGroup] 2024-11-16T05:55:05,367 INFO [M:0;3456ee6a3164:38307 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:55:05,367 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38307-0x10047194fa70000, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T05:55:05,367 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38307-0x10047194fa70000, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:55:05,367 INFO [M:0;3456ee6a3164:38307 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T05:55:05,368 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-16T05:55:05,368 DEBUG [M:0;3456ee6a3164:38307 {}] zookeeper.ZKUtil(347): master:38307-0x10047194fa70000, quorum=127.0.0.1:62345, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-16T05:55:05,368 WARN [M:0;3456ee6a3164:38307 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-16T05:55:05,369 INFO [M:0;3456ee6a3164:38307 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/.lastflushedseqids 2024-11-16T05:55:05,377 INFO [M:0;3456ee6a3164:38307 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T05:55:05,377 INFO [M:0;3456ee6a3164:38307 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-16T05:55:05,378 DEBUG [M:0;3456ee6a3164:38307 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T05:55:05,378 INFO [M:0;3456ee6a3164:38307 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T05:55:05,378 DEBUG [M:0;3456ee6a3164:38307 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T05:55:05,378 DEBUG [M:0;3456ee6a3164:38307 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T05:55:05,378 DEBUG [M:0;3456ee6a3164:38307 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T05:55:05,378 INFO [M:0;3456ee6a3164:38307 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.67 KB heapSize=43.02 KB 2024-11-16T05:55:05,388 DEBUG [M:0;3456ee6a3164:38307 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 2c3a58ff76a54c32a0535898ebd4af95 2024-11-16T05:55:05,402 DEBUG [M:0;3456ee6a3164:38307 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2c3a58ff76a54c32a0535898ebd4af95 is 82, key is hbase:meta,,1/info:regioninfo/1731736502345/Put/seqid=0 2024-11-16T05:55:05,403 INFO [M:0;3456ee6a3164:38307 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2c3a58ff76a54c32a0535898ebd4af95 2024-11-16T05:55:05,410 DEBUG [M:0;3456ee6a3164:38307 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on aa0a2f902d2840f8a5319ac5e3eedb09 2024-11-16T05:55:05,423 DEBUG [M:0;3456ee6a3164:38307 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/aa0a2f902d2840f8a5319ac5e3eedb09 is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731736503115/Put/seqid=0 2024-11-16T05:55:05,423 INFO [M:0;3456ee6a3164:38307 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.05 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/aa0a2f902d2840f8a5319ac5e3eedb09 2024-11-16T05:55:05,431 DEBUG [M:0;3456ee6a3164:38307 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 2ea2648d363e4432a5ba9de8e8508c93 2024-11-16T05:55:05,445 DEBUG [M:0;3456ee6a3164:38307 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2ea2648d363e4432a5ba9de8e8508c93 is 69, key is 3456ee6a3164,33069,1731736500303/rs:state/1731736501636/Put/seqid=0 2024-11-16T05:55:05,446 INFO [M:0;3456ee6a3164:38307 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2ea2648d363e4432a5ba9de8e8508c93 2024-11-16T05:55:05,449 DEBUG [M:0;3456ee6a3164:38307 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2c3a58ff76a54c32a0535898ebd4af95 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2c3a58ff76a54c32a0535898ebd4af95 2024-11-16T05:55:05,453 INFO [M:0;3456ee6a3164:38307 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2c3a58ff76a54c32a0535898ebd4af95, entries=8, sequenceid=84, filesize=5.5 K 2024-11-16T05:55:05,454 DEBUG [M:0;3456ee6a3164:38307 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/aa0a2f902d2840f8a5319ac5e3eedb09 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/aa0a2f902d2840f8a5319ac5e3eedb09 2024-11-16T05:55:05,456 INFO [M:0;3456ee6a3164:38307 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/aa0a2f902d2840f8a5319ac5e3eedb09, entries=9, sequenceid=84, filesize=6.5 K 2024-11-16T05:55:05,456 DEBUG [M:0;3456ee6a3164:38307 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2ea2648d363e4432a5ba9de8e8508c93 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2ea2648d363e4432a5ba9de8e8508c93 2024-11-16T05:55:05,456 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33069-0x10047194fa70001, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T05:55:05,456 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33069-0x10047194fa70001, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T05:55:05,457 INFO [RS:0;3456ee6a3164:33069 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T05:55:05,457 INFO [RS:0;3456ee6a3164:33069 {}] regionserver.HRegionServer(1031): Exiting; stopping=3456ee6a3164,33069,1731736500303; zookeeper connection closed. 2024-11-16T05:55:05,457 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@65875539 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@65875539 2024-11-16T05:55:05,458 INFO [M:0;3456ee6a3164:38307 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2ea2648d363e4432a5ba9de8e8508c93, entries=2, sequenceid=84, filesize=5.1 K 2024-11-16T05:55:05,459 INFO [M:0;3456ee6a3164:38307 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 81ms, sequenceid=84, compaction requested=false 2024-11-16T05:55:05,461 INFO [M:0;3456ee6a3164:38307 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T05:55:05,461 DEBUG [M:0;3456ee6a3164:38307 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731736505378Disabling compacts and flushes for region at 1731736505378Disabling writes for close at 1731736505378Obtaining lock to block concurrent updates at 1731736505378Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731736505378Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35506, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1731736505379 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731736505379Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731736505379Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731736505401 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731736505401Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731736505405 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731736505422 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731736505422Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731736505425 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731736505445 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731736505445Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3375a6a4: reopening flushed file at 1731736505449 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d455084: reopening flushed file at 1731736505453 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e9181f6: reopening flushed file at 1731736505456 (+3 ms)Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 81ms, sequenceid=84, compaction requested=false at 1731736505459 (+3 ms)Writing region close event to WAL at 1731736505461 (+2 ms)Closed at 1731736505461 2024-11-16T05:55:05,461 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36505-0x10047194fa70002, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T05:55:05,461 INFO [RS:1;3456ee6a3164:36505 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T05:55:05,461 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36505-0x10047194fa70002, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T05:55:05,461 INFO [RS:1;3456ee6a3164:36505 {}] regionserver.HRegionServer(1031): Exiting; stopping=3456ee6a3164,36505,1731736500481; zookeeper connection closed. 2024-11-16T05:55:05,465 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@209f84cc {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@209f84cc 2024-11-16T05:55:05,465 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-16T05:55:05,466 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T05:55:05,466 INFO [M:0;3456ee6a3164:38307 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-16T05:55:05,466 INFO [M:0;3456ee6a3164:38307 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38307 2024-11-16T05:55:05,466 INFO [M:0;3456ee6a3164:38307 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T05:55:05,568 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38307-0x10047194fa70000, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T05:55:05,568 INFO [M:0;3456ee6a3164:38307 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T05:55:05,568 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38307-0x10047194fa70000, quorum=127.0.0.1:62345, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T05:55:05,572 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-16T05:55:05,591 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-16T05:55:05,592 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T05:55:05,599 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=49 (was 12) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/3456ee6a3164:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731736505239 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/3456ee6a3164:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/3456ee6a3164:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/3456ee6a3164:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731736505244 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731736503516 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731736505064 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=280 (was 235) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 13), AvailableMemoryMB=4242 (was 5064) 2024-11-16T05:55:05,606 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=49, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=280, ProcessCount=11, AvailableMemoryMB=4242 2024-11-16T05:55:05,607 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/hadoop-log-dir so I do NOT create it in target/test-data/0f004a72-2e32-a29e-dcac-b939763ff566 2024-11-16T05:55:05,607 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/hadoop-tmp-dir so I do NOT create it in target/test-data/0f004a72-2e32-a29e-dcac-b939763ff566 2024-11-16T05:55:05,607 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T05:55:05,613 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0f004a72-2e32-a29e-dcac-b939763ff566/cluster_3478a130-ae7b-ec74-b878-04e1502087cf, deleteOnExit=true 2024-11-16T05:55:05,618 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0f004a72-2e32-a29e-dcac-b939763ff566/cluster_3478a130-ae7b-ec74-b878-04e1502087cf/zookeeper_0, clientPort=53959, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0f004a72-2e32-a29e-dcac-b939763ff566/cluster_3478a130-ae7b-ec74-b878-04e1502087cf/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0f004a72-2e32-a29e-dcac-b939763ff566/cluster_3478a130-ae7b-ec74-b878-04e1502087cf/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-16T05:55:05,623 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53959 2024-11-16T05:55:05,644 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926 with version=8 2024-11-16T05:55:05,644 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/hbase-staging 2024-11-16T05:55:05,648 INFO [Time-limited test {}] client.ConnectionUtils(128): master/3456ee6a3164:0 server-side Connection retries=45 2024-11-16T05:55:05,648 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T05:55:05,649 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T05:55:05,649 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T05:55:05,649 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T05:55:05,649 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T05:55:05,649 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-16T05:55:05,649 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T05:55:05,656 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44763 2024-11-16T05:55:05,659 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44763 connecting to ZooKeeper ensemble=127.0.0.1:53959 2024-11-16T05:55:05,668 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:447630x0, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T05:55:05,668 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44763-0x1004719697f0000 connected 2024-11-16T05:55:05,698 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T05:55:05,698 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T05:55:05,701 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44763-0x1004719697f0000, quorum=127.0.0.1:53959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T05:55:05,702 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926, hbase.cluster.distributed=false 2024-11-16T05:55:05,705 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44763-0x1004719697f0000, quorum=127.0.0.1:53959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T05:55:05,705 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44763 2024-11-16T05:55:05,705 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44763 2024-11-16T05:55:05,706 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44763 2024-11-16T05:55:05,706 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44763 2024-11-16T05:55:05,707 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44763 2024-11-16T05:55:05,726 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/3456ee6a3164:0 server-side Connection retries=45 2024-11-16T05:55:05,726 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T05:55:05,726 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T05:55:05,726 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T05:55:05,726 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T05:55:05,726 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T05:55:05,726 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T05:55:05,727 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T05:55:05,727 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44729 2024-11-16T05:55:05,729 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44729 connecting to ZooKeeper ensemble=127.0.0.1:53959 2024-11-16T05:55:05,730 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T05:55:05,730 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T05:55:05,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:447290x0, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T05:55:05,740 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:447290x0, quorum=127.0.0.1:53959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T05:55:05,741 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T05:55:05,742 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44729-0x1004719697f0001 connected 2024-11-16T05:55:05,745 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0f004a72-2e32-a29e-dcac-b939763ff566/bucket.cache 2024-11-16T05:55:05,746 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-16T05:55:05,746 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-16T05:55:05,746 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-16T05:55:05,749 INFO [Thread-119 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-16T05:55:05,749 WARN [Thread-119 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-16T05:55:05,749 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0f004a72-2e32-a29e-dcac-b939763ff566/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0f004a72-2e32-a29e-dcac-b939763ff566/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-16T05:55:05,749 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-16T05:55:05,749 INFO [Thread-119 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0f004a72-2e32-a29e-dcac-b939763ff566/bucket.persistence is complete. 2024-11-16T05:55:05,750 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T05:55:05,751 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44729-0x1004719697f0001, quorum=127.0.0.1:53959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T05:55:05,755 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44729-0x1004719697f0001, quorum=127.0.0.1:53959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T05:55:05,773 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44729 2024-11-16T05:55:05,797 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44729 2024-11-16T05:55:05,802 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44729 2024-11-16T05:55:05,805 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44729 2024-11-16T05:55:05,806 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44729 2024-11-16T05:55:05,830 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/3456ee6a3164:0 server-side Connection retries=45 2024-11-16T05:55:05,830 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T05:55:05,830 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T05:55:05,831 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T05:55:05,831 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T05:55:05,831 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T05:55:05,831 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T05:55:05,831 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T05:55:05,832 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40829 2024-11-16T05:55:05,835 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40829 connecting to ZooKeeper ensemble=127.0.0.1:53959 2024-11-16T05:55:05,836 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T05:55:05,836 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T05:55:05,841 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:408290x0, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T05:55:05,842 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:408290x0, quorum=127.0.0.1:53959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T05:55:05,843 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T05:55:05,849 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40829-0x1004719697f0002 connected 2024-11-16T05:55:05,853 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0f004a72-2e32-a29e-dcac-b939763ff566/bucket.cache 2024-11-16T05:55:05,853 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-16T05:55:05,853 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-16T05:55:05,853 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-16T05:55:05,856 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-16T05:55:05,856 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-16T05:55:05,857 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-16T05:55:05,857 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0f004a72-2e32-a29e-dcac-b939763ff566/bucket.persistence is complete. 2024-11-16T05:55:05,858 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0f004a72-2e32-a29e-dcac-b939763ff566/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0f004a72-2e32-a29e-dcac-b939763ff566/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-16T05:55:05,877 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T05:55:05,878 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40829-0x1004719697f0002, quorum=127.0.0.1:53959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T05:55:05,884 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40829-0x1004719697f0002, quorum=127.0.0.1:53959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T05:55:05,889 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40829 2024-11-16T05:55:05,890 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40829 2024-11-16T05:55:05,891 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40829 2024-11-16T05:55:05,898 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40829 2024-11-16T05:55:05,899 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40829 2024-11-16T05:55:05,918 DEBUG [M:0;3456ee6a3164:44763 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;3456ee6a3164:44763 2024-11-16T05:55:05,919 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/3456ee6a3164,44763,1731736505647 2024-11-16T05:55:05,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40829-0x1004719697f0002, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T05:55:05,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44763-0x1004719697f0000, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T05:55:05,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44729-0x1004719697f0001, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T05:55:05,925 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44763-0x1004719697f0000, quorum=127.0.0.1:53959, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/3456ee6a3164,44763,1731736505647 2024-11-16T05:55:05,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44763-0x1004719697f0000, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:55:05,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40829-0x1004719697f0002, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T05:55:05,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44729-0x1004719697f0001, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T05:55:05,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40829-0x1004719697f0002, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:55:05,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44729-0x1004719697f0001, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:55:05,930 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44763-0x1004719697f0000, quorum=127.0.0.1:53959, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-16T05:55:05,931 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/3456ee6a3164,44763,1731736505647 from backup master directory 2024-11-16T05:55:05,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44763-0x1004719697f0000, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/3456ee6a3164,44763,1731736505647 2024-11-16T05:55:05,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40829-0x1004719697f0002, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T05:55:05,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44729-0x1004719697f0001, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T05:55:05,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44763-0x1004719697f0000, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T05:55:05,933 WARN [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T05:55:05,933 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=3456ee6a3164,44763,1731736505647 2024-11-16T05:55:05,935 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/hbase.id] with ID: 8b126dd9-c05c-4c77-be92-82dc96bb59cc 2024-11-16T05:55:05,935 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/.tmp/hbase.id 2024-11-16T05:55:05,941 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/hbase.id] 2024-11-16T05:55:06,008 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T05:55:06,009 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-16T05:55:06,011 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-16T05:55:06,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40829-0x1004719697f0002, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:55:06,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44763-0x1004719697f0000, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:55:06,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44729-0x1004719697f0001, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:55:06,021 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T05:55:06,023 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-16T05:55:06,023 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T05:55:06,048 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/MasterData/data/master/store 2024-11-16T05:55:06,056 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:55:06,056 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T05:55:06,056 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T05:55:06,056 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T05:55:06,056 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T05:55:06,056 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T05:55:06,056 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T05:55:06,056 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731736506056Disabling compacts and flushes for region at 1731736506056Disabling writes for close at 1731736506056Writing region close event to WAL at 1731736506056Closed at 1731736506056 2024-11-16T05:55:06,056 WARN [master/3456ee6a3164:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/MasterData/data/master/store/.initializing 2024-11-16T05:55:06,056 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/MasterData/WALs/3456ee6a3164,44763,1731736505647 2024-11-16T05:55:06,063 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T05:55:06,066 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3456ee6a3164%2C44763%2C1731736505647, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/MasterData/WALs/3456ee6a3164,44763,1731736505647, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/MasterData/oldWALs, maxLogs=10 2024-11-16T05:55:06,087 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/MasterData/WALs/3456ee6a3164,44763,1731736505647/3456ee6a3164%2C44763%2C1731736505647.1731736506067 2024-11-16T05:55:06,088 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-16T05:55:06,088 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-16T05:55:06,093 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:55:06,093 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:55:06,093 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:55:06,094 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:55:06,098 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-16T05:55:06,098 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:55:06,098 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:55:06,098 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:55:06,101 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-16T05:55:06,101 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:55:06,101 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:55:06,102 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:55:06,104 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-16T05:55:06,104 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:55:06,105 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:55:06,105 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:55:06,108 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-16T05:55:06,108 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:55:06,108 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:55:06,108 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:55:06,108 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:55:06,108 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:55:06,109 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:55:06,109 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:55:06,109 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T05:55:06,109 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:55:06,114 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:55:06,115 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64122652, jitterRate=-0.0444980263710022}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T05:55:06,116 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731736506093Initializing all the Stores at 1731736506094 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731736506094Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736506094Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736506094Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736506094Cleaning up temporary data from old regions at 1731736506109 (+15 ms)Region opened successfully at 1731736506115 (+6 ms) 2024-11-16T05:55:06,116 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-16T05:55:06,119 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69db80ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3456ee6a3164/172.17.0.2:0 2024-11-16T05:55:06,120 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-16T05:55:06,120 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-16T05:55:06,121 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-16T05:55:06,121 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T05:55:06,121 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-16T05:55:06,121 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-16T05:55:06,121 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-16T05:55:06,123 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-16T05:55:06,124 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44763-0x1004719697f0000, quorum=127.0.0.1:53959, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-16T05:55:06,126 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-16T05:55:06,127 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-16T05:55:06,127 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44763-0x1004719697f0000, quorum=127.0.0.1:53959, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-16T05:55:06,128 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-16T05:55:06,128 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-16T05:55:06,130 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44763-0x1004719697f0000, quorum=127.0.0.1:53959, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-16T05:55:06,130 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-16T05:55:06,131 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44763-0x1004719697f0000, quorum=127.0.0.1:53959, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-16T05:55:06,132 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-16T05:55:06,134 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44763-0x1004719697f0000, quorum=127.0.0.1:53959, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-16T05:55:06,135 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-16T05:55:06,136 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44729-0x1004719697f0001, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T05:55:06,136 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40829-0x1004719697f0002, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T05:55:06,136 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44729-0x1004719697f0001, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:55:06,136 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40829-0x1004719697f0002, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:55:06,136 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44763-0x1004719697f0000, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T05:55:06,137 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44763-0x1004719697f0000, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:55:06,137 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=3456ee6a3164,44763,1731736505647, sessionid=0x1004719697f0000, setting cluster-up flag (Was=false) 2024-11-16T05:55:06,139 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44763-0x1004719697f0000, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:55:06,139 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40829-0x1004719697f0002, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:55:06,139 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44729-0x1004719697f0001, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:55:06,143 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-16T05:55:06,144 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=3456ee6a3164,44763,1731736505647 2024-11-16T05:55:06,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44763-0x1004719697f0000, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:55:06,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44729-0x1004719697f0001, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:55:06,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40829-0x1004719697f0002, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:55:06,149 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-16T05:55:06,151 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=3456ee6a3164,44763,1731736505647 2024-11-16T05:55:06,151 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-16T05:55:06,152 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T05:55:06,153 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-16T05:55:06,153 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-16T05:55:06,153 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 3456ee6a3164,44763,1731736505647 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-16T05:55:06,155 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/3456ee6a3164:0, corePoolSize=5, maxPoolSize=5 2024-11-16T05:55:06,155 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/3456ee6a3164:0, corePoolSize=5, maxPoolSize=5 2024-11-16T05:55:06,155 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/3456ee6a3164:0, corePoolSize=5, maxPoolSize=5 2024-11-16T05:55:06,155 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/3456ee6a3164:0, corePoolSize=5, maxPoolSize=5 2024-11-16T05:55:06,155 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/3456ee6a3164:0, corePoolSize=10, maxPoolSize=10 2024-11-16T05:55:06,155 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:06,155 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/3456ee6a3164:0, corePoolSize=2, maxPoolSize=2 2024-11-16T05:55:06,156 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:06,158 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T05:55:06,158 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-16T05:55:06,158 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:55:06,159 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-16T05:55:06,165 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731736536165 2024-11-16T05:55:06,165 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-16T05:55:06,165 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-16T05:55:06,165 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-16T05:55:06,165 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-16T05:55:06,165 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-16T05:55:06,166 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-16T05:55:06,170 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:06,173 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-16T05:55:06,173 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-16T05:55:06,173 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-16T05:55:06,174 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-16T05:55:06,174 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-16T05:55:06,174 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/3456ee6a3164:0:becomeActiveMaster-HFileCleaner.large.0-1731736506174,5,FailOnTimeoutGroup] 2024-11-16T05:55:06,177 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/3456ee6a3164:0:becomeActiveMaster-HFileCleaner.small.0-1731736506174,5,FailOnTimeoutGroup] 2024-11-16T05:55:06,177 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:06,177 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-16T05:55:06,177 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:06,177 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:06,182 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-16T05:55:06,182 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926 2024-11-16T05:55:06,192 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:55:06,193 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-16T05:55:06,197 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T05:55:06,197 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:55:06,197 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:55:06,198 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-16T05:55:06,202 INFO [RS:0;3456ee6a3164:44729 {}] regionserver.HRegionServer(746): ClusterId : 8b126dd9-c05c-4c77-be92-82dc96bb59cc 2024-11-16T05:55:06,202 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T05:55:06,202 DEBUG [RS:0;3456ee6a3164:44729 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T05:55:06,202 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:55:06,202 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:55:06,202 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-16T05:55:06,204 DEBUG [RS:0;3456ee6a3164:44729 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T05:55:06,204 DEBUG [RS:0;3456ee6a3164:44729 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T05:55:06,206 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T05:55:06,206 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:55:06,206 INFO [RS:1;3456ee6a3164:40829 {}] regionserver.HRegionServer(746): ClusterId : 8b126dd9-c05c-4c77-be92-82dc96bb59cc 2024-11-16T05:55:06,206 DEBUG [RS:1;3456ee6a3164:40829 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T05:55:06,207 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:55:06,207 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-16T05:55:06,208 DEBUG [RS:0;3456ee6a3164:44729 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T05:55:06,209 DEBUG [RS:0;3456ee6a3164:44729 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d5fd14b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3456ee6a3164/172.17.0.2:0 2024-11-16T05:55:06,209 DEBUG [RS:1;3456ee6a3164:40829 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T05:55:06,210 DEBUG [RS:1;3456ee6a3164:40829 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T05:55:06,212 DEBUG [RS:1;3456ee6a3164:40829 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T05:55:06,213 DEBUG [RS:1;3456ee6a3164:40829 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6eaf97d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3456ee6a3164/172.17.0.2:0 2024-11-16T05:55:06,214 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T05:55:06,214 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:55:06,214 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:55:06,214 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T05:55:06,215 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740 2024-11-16T05:55:06,215 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740 2024-11-16T05:55:06,216 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T05:55:06,216 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T05:55:06,216 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T05:55:06,219 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T05:55:06,230 DEBUG [RS:1;3456ee6a3164:40829 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;3456ee6a3164:40829 2024-11-16T05:55:06,230 INFO [RS:1;3456ee6a3164:40829 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T05:55:06,230 INFO [RS:1;3456ee6a3164:40829 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T05:55:06,230 DEBUG [RS:1;3456ee6a3164:40829 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T05:55:06,231 DEBUG [RS:0;3456ee6a3164:44729 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;3456ee6a3164:44729 2024-11-16T05:55:06,232 INFO [RS:0;3456ee6a3164:44729 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T05:55:06,232 INFO [RS:0;3456ee6a3164:44729 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T05:55:06,232 DEBUG [RS:0;3456ee6a3164:44729 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T05:55:06,232 INFO [RS:1;3456ee6a3164:40829 {}] regionserver.HRegionServer(2659): reportForDuty to master=3456ee6a3164,44763,1731736505647 with port=40829, startcode=1731736505829 2024-11-16T05:55:06,232 DEBUG [RS:1;3456ee6a3164:40829 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T05:55:06,234 INFO [RS:0;3456ee6a3164:44729 {}] regionserver.HRegionServer(2659): reportForDuty to master=3456ee6a3164,44763,1731736505647 with port=44729, startcode=1731736505725 2024-11-16T05:55:06,234 DEBUG [RS:0;3456ee6a3164:44729 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T05:55:06,238 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56469, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T05:55:06,239 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44763 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 3456ee6a3164,40829,1731736505829 2024-11-16T05:55:06,239 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44763 {}] master.ServerManager(517): Registering regionserver=3456ee6a3164,40829,1731736505829 2024-11-16T05:55:06,242 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46747, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T05:55:06,243 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44763 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 3456ee6a3164,44729,1731736505725 2024-11-16T05:55:06,243 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44763 {}] master.ServerManager(517): Registering regionserver=3456ee6a3164,44729,1731736505725 2024-11-16T05:55:06,245 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:55:06,245 DEBUG [RS:1;3456ee6a3164:40829 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926 2024-11-16T05:55:06,245 DEBUG [RS:1;3456ee6a3164:40829 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-16T05:55:06,245 DEBUG [RS:1;3456ee6a3164:40829 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T05:55:06,245 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72029224, jitterRate=0.07331907749176025}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T05:55:06,245 DEBUG [RS:0;3456ee6a3164:44729 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926 2024-11-16T05:55:06,246 DEBUG [RS:0;3456ee6a3164:44729 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-16T05:55:06,246 DEBUG [RS:0;3456ee6a3164:44729 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T05:55:06,246 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731736506192Initializing all the Stores at 1731736506192Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731736506192Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731736506193 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736506193Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731736506193Cleaning up temporary data from old regions at 1731736506216 (+23 ms)Region opened successfully at 1731736506246 (+30 ms) 2024-11-16T05:55:06,246 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T05:55:06,246 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T05:55:06,246 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T05:55:06,246 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T05:55:06,247 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T05:55:06,247 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T05:55:06,247 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731736506246Disabling compacts and flushes for region at 1731736506246Disabling writes for close at 1731736506246Writing region close event to WAL at 1731736506247 (+1 ms)Closed at 1731736506247 2024-11-16T05:55:06,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44763-0x1004719697f0000, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T05:55:06,248 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T05:55:06,248 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-16T05:55:06,249 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-16T05:55:06,256 DEBUG [RS:0;3456ee6a3164:44729 {}] zookeeper.ZKUtil(111): regionserver:44729-0x1004719697f0001, quorum=127.0.0.1:53959, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/3456ee6a3164,44729,1731736505725 2024-11-16T05:55:06,256 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T05:55:06,256 WARN [RS:0;3456ee6a3164:44729 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T05:55:06,256 INFO [RS:0;3456ee6a3164:44729 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T05:55:06,256 DEBUG [RS:0;3456ee6a3164:44729 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/WALs/3456ee6a3164,44729,1731736505725 2024-11-16T05:55:06,256 DEBUG [RS:1;3456ee6a3164:40829 {}] zookeeper.ZKUtil(111): regionserver:40829-0x1004719697f0002, quorum=127.0.0.1:53959, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/3456ee6a3164,40829,1731736505829 2024-11-16T05:55:06,256 WARN [RS:1;3456ee6a3164:40829 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T05:55:06,256 INFO [RS:1;3456ee6a3164:40829 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T05:55:06,257 DEBUG [RS:1;3456ee6a3164:40829 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/WALs/3456ee6a3164,40829,1731736505829 2024-11-16T05:55:06,257 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [3456ee6a3164,40829,1731736505829] 2024-11-16T05:55:06,257 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [3456ee6a3164,44729,1731736505725] 2024-11-16T05:55:06,257 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-16T05:55:06,285 INFO [RS:1;3456ee6a3164:40829 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T05:55:06,285 INFO [RS:0;3456ee6a3164:44729 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T05:55:06,287 INFO [RS:0;3456ee6a3164:44729 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T05:55:06,288 INFO [RS:0;3456ee6a3164:44729 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T05:55:06,288 INFO [RS:0;3456ee6a3164:44729 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:06,288 INFO [RS:0;3456ee6a3164:44729 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T05:55:06,289 INFO [RS:0;3456ee6a3164:44729 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T05:55:06,289 INFO [RS:0;3456ee6a3164:44729 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:06,289 DEBUG [RS:0;3456ee6a3164:44729 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:06,289 DEBUG [RS:0;3456ee6a3164:44729 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:06,289 DEBUG [RS:0;3456ee6a3164:44729 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:06,289 DEBUG [RS:0;3456ee6a3164:44729 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:06,289 DEBUG [RS:0;3456ee6a3164:44729 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:06,289 DEBUG [RS:0;3456ee6a3164:44729 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0, corePoolSize=2, maxPoolSize=2 2024-11-16T05:55:06,289 DEBUG [RS:0;3456ee6a3164:44729 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:06,290 DEBUG [RS:0;3456ee6a3164:44729 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:06,290 DEBUG [RS:0;3456ee6a3164:44729 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:06,290 DEBUG [RS:0;3456ee6a3164:44729 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:06,290 DEBUG [RS:0;3456ee6a3164:44729 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:06,290 DEBUG [RS:0;3456ee6a3164:44729 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:06,290 DEBUG [RS:0;3456ee6a3164:44729 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/3456ee6a3164:0, corePoolSize=3, maxPoolSize=3 2024-11-16T05:55:06,290 DEBUG [RS:0;3456ee6a3164:44729 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0, corePoolSize=3, maxPoolSize=3 2024-11-16T05:55:06,293 INFO [RS:1;3456ee6a3164:40829 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T05:55:06,297 INFO [RS:1;3456ee6a3164:40829 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T05:55:06,297 INFO [RS:0;3456ee6a3164:44729 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:06,297 INFO [RS:1;3456ee6a3164:40829 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:06,298 INFO [RS:0;3456ee6a3164:44729 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:06,298 INFO [RS:0;3456ee6a3164:44729 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:06,298 INFO [RS:0;3456ee6a3164:44729 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:06,298 INFO [RS:0;3456ee6a3164:44729 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:06,298 INFO [RS:0;3456ee6a3164:44729 {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,44729,1731736505725-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T05:55:06,301 INFO [RS:1;3456ee6a3164:40829 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T05:55:06,302 INFO [RS:1;3456ee6a3164:40829 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T05:55:06,302 INFO [RS:1;3456ee6a3164:40829 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:06,302 DEBUG [RS:1;3456ee6a3164:40829 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:06,302 DEBUG [RS:1;3456ee6a3164:40829 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:06,302 DEBUG [RS:1;3456ee6a3164:40829 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:06,302 DEBUG [RS:1;3456ee6a3164:40829 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:06,302 DEBUG [RS:1;3456ee6a3164:40829 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:06,302 DEBUG [RS:1;3456ee6a3164:40829 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0, corePoolSize=2, maxPoolSize=2 2024-11-16T05:55:06,303 DEBUG [RS:1;3456ee6a3164:40829 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:06,303 DEBUG [RS:1;3456ee6a3164:40829 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:06,303 DEBUG [RS:1;3456ee6a3164:40829 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:06,303 DEBUG [RS:1;3456ee6a3164:40829 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:06,303 DEBUG [RS:1;3456ee6a3164:40829 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:06,303 DEBUG [RS:1;3456ee6a3164:40829 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:06,303 DEBUG [RS:1;3456ee6a3164:40829 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/3456ee6a3164:0, corePoolSize=3, maxPoolSize=3 2024-11-16T05:55:06,303 DEBUG [RS:1;3456ee6a3164:40829 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0, corePoolSize=3, maxPoolSize=3 2024-11-16T05:55:06,310 INFO [RS:1;3456ee6a3164:40829 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:06,310 INFO [RS:1;3456ee6a3164:40829 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:06,310 INFO [RS:1;3456ee6a3164:40829 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:06,310 INFO [RS:1;3456ee6a3164:40829 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:06,310 INFO [RS:1;3456ee6a3164:40829 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:06,310 INFO [RS:1;3456ee6a3164:40829 {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,40829,1731736505829-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T05:55:06,320 INFO [RS:0;3456ee6a3164:44729 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T05:55:06,320 INFO [RS:0;3456ee6a3164:44729 {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,44729,1731736505725-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:06,320 INFO [RS:0;3456ee6a3164:44729 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:06,320 INFO [RS:0;3456ee6a3164:44729 {}] regionserver.Replication(171): 3456ee6a3164,44729,1731736505725 started 2024-11-16T05:55:06,331 INFO [RS:1;3456ee6a3164:40829 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T05:55:06,331 INFO [RS:1;3456ee6a3164:40829 {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,40829,1731736505829-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:06,331 INFO [RS:1;3456ee6a3164:40829 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:06,332 INFO [RS:1;3456ee6a3164:40829 {}] regionserver.Replication(171): 3456ee6a3164,40829,1731736505829 started 2024-11-16T05:55:06,339 INFO [RS:0;3456ee6a3164:44729 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:06,340 INFO [RS:0;3456ee6a3164:44729 {}] regionserver.HRegionServer(1482): Serving as 3456ee6a3164,44729,1731736505725, RpcServer on 3456ee6a3164/172.17.0.2:44729, sessionid=0x1004719697f0001 2024-11-16T05:55:06,340 DEBUG [RS:0;3456ee6a3164:44729 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T05:55:06,340 DEBUG [RS:0;3456ee6a3164:44729 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 3456ee6a3164,44729,1731736505725 2024-11-16T05:55:06,340 DEBUG [RS:0;3456ee6a3164:44729 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3456ee6a3164,44729,1731736505725' 2024-11-16T05:55:06,340 DEBUG [RS:0;3456ee6a3164:44729 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T05:55:06,341 DEBUG [RS:0;3456ee6a3164:44729 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T05:55:06,341 DEBUG [RS:0;3456ee6a3164:44729 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T05:55:06,341 DEBUG [RS:0;3456ee6a3164:44729 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T05:55:06,341 DEBUG [RS:0;3456ee6a3164:44729 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 3456ee6a3164,44729,1731736505725 2024-11-16T05:55:06,341 DEBUG [RS:0;3456ee6a3164:44729 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3456ee6a3164,44729,1731736505725' 2024-11-16T05:55:06,341 DEBUG [RS:0;3456ee6a3164:44729 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T05:55:06,342 DEBUG [RS:0;3456ee6a3164:44729 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T05:55:06,342 DEBUG [RS:0;3456ee6a3164:44729 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T05:55:06,342 INFO [RS:0;3456ee6a3164:44729 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T05:55:06,342 INFO [RS:0;3456ee6a3164:44729 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T05:55:06,348 INFO [RS:1;3456ee6a3164:40829 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:06,348 INFO [RS:1;3456ee6a3164:40829 {}] regionserver.HRegionServer(1482): Serving as 3456ee6a3164,40829,1731736505829, RpcServer on 3456ee6a3164/172.17.0.2:40829, sessionid=0x1004719697f0002 2024-11-16T05:55:06,348 DEBUG [RS:1;3456ee6a3164:40829 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T05:55:06,349 DEBUG [RS:1;3456ee6a3164:40829 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 3456ee6a3164,40829,1731736505829 2024-11-16T05:55:06,349 DEBUG [RS:1;3456ee6a3164:40829 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3456ee6a3164,40829,1731736505829' 2024-11-16T05:55:06,349 DEBUG [RS:1;3456ee6a3164:40829 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T05:55:06,349 DEBUG [RS:1;3456ee6a3164:40829 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T05:55:06,350 DEBUG [RS:1;3456ee6a3164:40829 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T05:55:06,350 DEBUG [RS:1;3456ee6a3164:40829 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T05:55:06,350 DEBUG [RS:1;3456ee6a3164:40829 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 3456ee6a3164,40829,1731736505829 2024-11-16T05:55:06,350 DEBUG [RS:1;3456ee6a3164:40829 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3456ee6a3164,40829,1731736505829' 2024-11-16T05:55:06,350 DEBUG [RS:1;3456ee6a3164:40829 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T05:55:06,350 DEBUG [RS:1;3456ee6a3164:40829 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T05:55:06,351 DEBUG [RS:1;3456ee6a3164:40829 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T05:55:06,351 INFO [RS:1;3456ee6a3164:40829 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T05:55:06,351 INFO [RS:1;3456ee6a3164:40829 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T05:55:06,407 WARN [3456ee6a3164:44763 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-16T05:55:06,444 INFO [RS:0;3456ee6a3164:44729 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T05:55:06,445 INFO [RS:0;3456ee6a3164:44729 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=3456ee6a3164%2C44729%2C1731736505725, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/WALs/3456ee6a3164,44729,1731736505725, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/oldWALs, maxLogs=55 2024-11-16T05:55:06,452 INFO [RS:1;3456ee6a3164:40829 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T05:55:06,454 INFO [RS:1;3456ee6a3164:40829 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=3456ee6a3164%2C40829%2C1731736505829, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/WALs/3456ee6a3164,40829,1731736505829, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/oldWALs, maxLogs=55 2024-11-16T05:55:06,471 INFO [RS:0;3456ee6a3164:44729 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/WALs/3456ee6a3164,44729,1731736505725/3456ee6a3164%2C44729%2C1731736505725.1731736506449 2024-11-16T05:55:06,472 DEBUG [RS:0;3456ee6a3164:44729 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-16T05:55:06,475 INFO [RS:1;3456ee6a3164:40829 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/WALs/3456ee6a3164,40829,1731736505829/3456ee6a3164%2C40829%2C1731736505829.1731736506456 2024-11-16T05:55:06,475 DEBUG [RS:1;3456ee6a3164:40829 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-16T05:55:06,658 DEBUG [3456ee6a3164:44763 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-16T05:55:06,658 DEBUG [3456ee6a3164:44763 {}] balancer.BalancerClusterState(204): Hosts are {3456ee6a3164=0} racks are {/default-rack=0} 2024-11-16T05:55:06,659 DEBUG [3456ee6a3164:44763 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-16T05:55:06,659 DEBUG [3456ee6a3164:44763 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-16T05:55:06,659 DEBUG [3456ee6a3164:44763 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-16T05:55:06,659 DEBUG [3456ee6a3164:44763 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-16T05:55:06,659 INFO [3456ee6a3164:44763 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-16T05:55:06,659 INFO [3456ee6a3164:44763 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-16T05:55:06,659 DEBUG [3456ee6a3164:44763 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-16T05:55:06,659 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=3456ee6a3164,40829,1731736505829 2024-11-16T05:55:06,661 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3456ee6a3164,40829,1731736505829, state=OPENING 2024-11-16T05:55:06,662 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-16T05:55:06,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44763-0x1004719697f0000, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:55:06,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44729-0x1004719697f0001, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:55:06,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40829-0x1004719697f0002, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:55:06,665 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T05:55:06,665 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:55:06,666 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:55:06,666 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:55:06,666 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,40829,1731736505829}] 2024-11-16T05:55:06,821 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T05:55:06,824 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44361, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T05:55:06,827 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-16T05:55:06,827 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T05:55:06,827 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-16T05:55:06,828 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=3456ee6a3164%2C40829%2C1731736505829.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/WALs/3456ee6a3164,40829,1731736505829, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/oldWALs, maxLogs=55 2024-11-16T05:55:06,857 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/WALs/3456ee6a3164,40829,1731736505829/3456ee6a3164%2C40829%2C1731736505829.meta.1731736506829.meta 2024-11-16T05:55:06,861 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-16T05:55:06,861 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-16T05:55:06,862 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-16T05:55:06,862 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-16T05:55:06,862 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-16T05:55:06,862 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-16T05:55:06,862 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:55:06,862 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-16T05:55:06,862 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-16T05:55:06,863 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-16T05:55:06,864 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T05:55:06,864 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:55:06,865 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:55:06,865 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-16T05:55:06,866 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T05:55:06,866 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:55:06,866 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:55:06,866 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-16T05:55:06,867 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T05:55:06,867 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:55:06,867 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:55:06,868 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-16T05:55:06,868 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T05:55:06,868 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:55:06,869 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:55:06,869 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T05:55:06,869 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740 2024-11-16T05:55:06,870 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740 2024-11-16T05:55:06,870 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T05:55:06,870 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T05:55:06,870 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T05:55:06,871 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T05:55:06,871 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65891250, jitterRate=-0.01814386248588562}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T05:55:06,872 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-16T05:55:06,872 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731736506863Writing region info on filesystem at 1731736506863Initializing all the Stores at 1731736506863Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731736506863Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731736506863Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736506863Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731736506863Cleaning up temporary data from old regions at 1731736506870 (+7 ms)Running coprocessor post-open hooks at 1731736506872 (+2 ms)Region opened successfully at 1731736506872 2024-11-16T05:55:06,873 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731736506819 2024-11-16T05:55:06,876 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-16T05:55:06,876 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-16T05:55:06,876 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,40829,1731736505829 2024-11-16T05:55:06,877 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3456ee6a3164,40829,1731736505829, state=OPEN 2024-11-16T05:55:06,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40829-0x1004719697f0002, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:55:06,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44763-0x1004719697f0000, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:55:06,880 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:55:06,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44729-0x1004719697f0001, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:55:06,880 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:55:06,880 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:55:06,881 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=3456ee6a3164,40829,1731736505829 2024-11-16T05:55:06,884 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-16T05:55:06,884 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,40829,1731736505829 in 217 msec 2024-11-16T05:55:06,887 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-16T05:55:06,887 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 635 msec 2024-11-16T05:55:06,888 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T05:55:06,888 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-16T05:55:06,891 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:55:06,891 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,40829,1731736505829, seqNum=-1] 2024-11-16T05:55:06,892 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:55:06,893 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44535, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:55:06,903 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 749 msec 2024-11-16T05:55:06,903 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731736506903, completionTime=-1 2024-11-16T05:55:06,903 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-16T05:55:06,903 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T05:55:06,910 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-16T05:55:06,910 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731736566910 2024-11-16T05:55:06,910 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731736626910 2024-11-16T05:55:06,910 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-11-16T05:55:06,911 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-16T05:55:06,913 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,44763,1731736505647-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:06,913 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,44763,1731736505647-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:06,914 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,44763,1731736505647-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:06,914 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-3456ee6a3164:44763, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:06,914 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:06,918 DEBUG [master/3456ee6a3164:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-16T05:55:06,921 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:06,928 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.995sec 2024-11-16T05:55:06,929 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-16T05:55:06,929 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-16T05:55:06,929 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-16T05:55:06,929 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-16T05:55:06,929 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-16T05:55:06,929 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,44763,1731736505647-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T05:55:06,929 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,44763,1731736505647-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-16T05:55:06,933 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-16T05:55:06,933 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-16T05:55:06,933 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,44763,1731736505647-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:07,033 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dccad78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:55:07,034 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,44763,-1 for getting cluster id 2024-11-16T05:55:07,034 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:55:07,041 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b126dd9-c05c-4c77-be92-82dc96bb59cc' 2024-11-16T05:55:07,043 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:55:07,045 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b126dd9-c05c-4c77-be92-82dc96bb59cc" 2024-11-16T05:55:07,047 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@785b0786, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:55:07,047 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,44763,-1] 2024-11-16T05:55:07,047 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:55:07,048 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:55:07,060 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40668, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:55:07,063 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d4294a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:55:07,064 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:55:07,070 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,40829,1731736505829, seqNum=-1] 2024-11-16T05:55:07,071 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:55:07,100 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59416, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:55:07,110 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=3456ee6a3164,44763,1731736505647 2024-11-16T05:55:07,110 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T05:55:07,113 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 3456ee6a3164,44763,1731736505647 2024-11-16T05:55:07,114 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1bcf520a 2024-11-16T05:55:07,114 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T05:55:07,124 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40680, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T05:55:07,126 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44763 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1731736507110', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T05:55:07,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44763 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731736507110 2024-11-16T05:55:07,128 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731736507110 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T05:55:07,129 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:55:07,129 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44763 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1731736507110" procId is: 4 2024-11-16T05:55:07,130 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731736507110 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T05:55:07,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44763 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T05:55:07,188 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731736507110-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 49ee9ebdc7532df078f1ab2a638cddfd, NAME => 'testBlockEvictionOnGracefulStop1731736507110,,1731736507125.49ee9ebdc7532df078f1ab2a638cddfd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1731736507110', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926 2024-11-16T05:55:07,226 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731736507110-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731736507110,,1731736507125.49ee9ebdc7532df078f1ab2a638cddfd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:55:07,226 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731736507110-pool-0 {}] regionserver.HRegion(1722): Closing 49ee9ebdc7532df078f1ab2a638cddfd, disabling compactions & flushes 2024-11-16T05:55:07,226 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731736507110-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731736507110,,1731736507125.49ee9ebdc7532df078f1ab2a638cddfd. 2024-11-16T05:55:07,226 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731736507110-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731736507110,,1731736507125.49ee9ebdc7532df078f1ab2a638cddfd. 2024-11-16T05:55:07,226 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731736507110-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731736507110,,1731736507125.49ee9ebdc7532df078f1ab2a638cddfd. after waiting 0 ms 2024-11-16T05:55:07,226 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731736507110-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731736507110,,1731736507125.49ee9ebdc7532df078f1ab2a638cddfd. 2024-11-16T05:55:07,226 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731736507110-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731736507110,,1731736507125.49ee9ebdc7532df078f1ab2a638cddfd. 2024-11-16T05:55:07,226 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731736507110-pool-0 {}] regionserver.HRegion(1676): Region close journal for 49ee9ebdc7532df078f1ab2a638cddfd: Waiting for close lock at 1731736507226Disabling compacts and flushes for region at 1731736507226Disabling writes for close at 1731736507226Writing region close event to WAL at 1731736507226Closed at 1731736507226 2024-11-16T05:55:07,228 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731736507110 execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T05:55:07,228 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1731736507110,,1731736507125.49ee9ebdc7532df078f1ab2a638cddfd.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1731736507228"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736507228"}]},"ts":"1731736507228"} 2024-11-16T05:55:07,233 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T05:55:07,236 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731736507110 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T05:55:07,236 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731736507110","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731736507236"}]},"ts":"1731736507236"} 2024-11-16T05:55:07,240 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731736507110, state=ENABLING in hbase:meta 2024-11-16T05:55:07,241 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {3456ee6a3164=0} racks are {/default-rack=0} 2024-11-16T05:55:07,241 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-16T05:55:07,241 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-16T05:55:07,242 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-16T05:55:07,242 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-16T05:55:07,242 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-16T05:55:07,242 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-16T05:55:07,242 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-16T05:55:07,242 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731736507110, region=49ee9ebdc7532df078f1ab2a638cddfd, ASSIGN}] 2024-11-16T05:55:07,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44763 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T05:55:07,247 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731736507110, region=49ee9ebdc7532df078f1ab2a638cddfd, ASSIGN 2024-11-16T05:55:07,249 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731736507110, region=49ee9ebdc7532df078f1ab2a638cddfd, ASSIGN; state=OFFLINE, location=3456ee6a3164,40829,1731736505829; forceNewPlan=false, retain=false 2024-11-16T05:55:07,401 INFO [3456ee6a3164:44763 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-16T05:55:07,401 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=49ee9ebdc7532df078f1ab2a638cddfd, regionState=OPENING, regionLocation=3456ee6a3164,40829,1731736505829 2024-11-16T05:55:07,405 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731736507110, region=49ee9ebdc7532df078f1ab2a638cddfd, ASSIGN because future has completed 2024-11-16T05:55:07,405 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 49ee9ebdc7532df078f1ab2a638cddfd, server=3456ee6a3164,40829,1731736505829}] 2024-11-16T05:55:07,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44763 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T05:55:07,562 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731736507110,,1731736507125.49ee9ebdc7532df078f1ab2a638cddfd. 2024-11-16T05:55:07,562 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 49ee9ebdc7532df078f1ab2a638cddfd, NAME => 'testBlockEvictionOnGracefulStop1731736507110,,1731736507125.49ee9ebdc7532df078f1ab2a638cddfd.', STARTKEY => '', ENDKEY => ''} 2024-11-16T05:55:07,563 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731736507110 49ee9ebdc7532df078f1ab2a638cddfd 2024-11-16T05:55:07,563 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731736507110,,1731736507125.49ee9ebdc7532df078f1ab2a638cddfd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:55:07,563 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 49ee9ebdc7532df078f1ab2a638cddfd 2024-11-16T05:55:07,563 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 49ee9ebdc7532df078f1ab2a638cddfd 2024-11-16T05:55:07,564 INFO [StoreOpener-49ee9ebdc7532df078f1ab2a638cddfd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 49ee9ebdc7532df078f1ab2a638cddfd 2024-11-16T05:55:07,570 INFO [StoreOpener-49ee9ebdc7532df078f1ab2a638cddfd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 49ee9ebdc7532df078f1ab2a638cddfd columnFamilyName family 2024-11-16T05:55:07,570 DEBUG [StoreOpener-49ee9ebdc7532df078f1ab2a638cddfd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:55:07,570 INFO [StoreOpener-49ee9ebdc7532df078f1ab2a638cddfd-1 {}] regionserver.HStore(327): Store=49ee9ebdc7532df078f1ab2a638cddfd/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:55:07,570 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 49ee9ebdc7532df078f1ab2a638cddfd 2024-11-16T05:55:07,571 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/default/testBlockEvictionOnGracefulStop1731736507110/49ee9ebdc7532df078f1ab2a638cddfd 2024-11-16T05:55:07,571 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/default/testBlockEvictionOnGracefulStop1731736507110/49ee9ebdc7532df078f1ab2a638cddfd 2024-11-16T05:55:07,571 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 49ee9ebdc7532df078f1ab2a638cddfd 2024-11-16T05:55:07,571 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 49ee9ebdc7532df078f1ab2a638cddfd 2024-11-16T05:55:07,572 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 49ee9ebdc7532df078f1ab2a638cddfd 2024-11-16T05:55:07,577 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/default/testBlockEvictionOnGracefulStop1731736507110/49ee9ebdc7532df078f1ab2a638cddfd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:55:07,577 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 49ee9ebdc7532df078f1ab2a638cddfd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70030653, jitterRate=0.04353804886341095}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T05:55:07,577 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 49ee9ebdc7532df078f1ab2a638cddfd 2024-11-16T05:55:07,578 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 49ee9ebdc7532df078f1ab2a638cddfd: Running coprocessor pre-open hook at 1731736507563Writing region info on filesystem at 1731736507563Initializing all the Stores at 1731736507564 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736507564Cleaning up temporary data from old regions at 1731736507571 (+7 ms)Running coprocessor post-open hooks at 1731736507577 (+6 ms)Region opened successfully at 1731736507578 (+1 ms) 2024-11-16T05:55:07,579 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731736507110,,1731736507125.49ee9ebdc7532df078f1ab2a638cddfd., pid=6, masterSystemTime=1731736507559 2024-11-16T05:55:07,581 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731736507110,,1731736507125.49ee9ebdc7532df078f1ab2a638cddfd. 2024-11-16T05:55:07,581 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731736507110,,1731736507125.49ee9ebdc7532df078f1ab2a638cddfd. 2024-11-16T05:55:07,582 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=49ee9ebdc7532df078f1ab2a638cddfd, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,40829,1731736505829 2024-11-16T05:55:07,584 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 49ee9ebdc7532df078f1ab2a638cddfd, server=3456ee6a3164,40829,1731736505829 because future has completed 2024-11-16T05:55:07,588 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-16T05:55:07,588 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 49ee9ebdc7532df078f1ab2a638cddfd, server=3456ee6a3164,40829,1731736505829 in 181 msec 2024-11-16T05:55:07,590 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-16T05:55:07,590 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731736507110, region=49ee9ebdc7532df078f1ab2a638cddfd, ASSIGN in 346 msec 2024-11-16T05:55:07,590 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731736507110 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T05:55:07,591 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731736507110","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731736507590"}]},"ts":"1731736507590"} 2024-11-16T05:55:07,593 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731736507110, state=ENABLED in hbase:meta 2024-11-16T05:55:07,594 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731736507110 execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T05:55:07,596 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731736507110 in 468 msec 2024-11-16T05:55:07,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44763 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T05:55:07,763 INFO [RPCClient-NioEventLoopGroup-5-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1731736507110 completed 2024-11-16T05:55:07,763 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1731736507110 get assigned. Timeout = 60000ms 2024-11-16T05:55:07,763 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T05:55:07,768 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1731736507110 assigned to meta. Checking AM states. 2024-11-16T05:55:07,768 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T05:55:07,769 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1731736507110 assigned. 2024-11-16T05:55:07,774 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1731736507110', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1731736507110,,1731736507125.49ee9ebdc7532df078f1ab2a638cddfd., hostname=3456ee6a3164,40829,1731736505829, seqNum=2] 2024-11-16T05:55:07,780 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 49ee9ebdc7532df078f1ab2a638cddfd 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-16T05:55:07,789 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 61d72c4def6246ff80a656f7c8dae318 2024-11-16T05:55:07,800 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/default/testBlockEvictionOnGracefulStop1731736507110/49ee9ebdc7532df078f1ab2a638cddfd/.tmp/family/61d72c4def6246ff80a656f7c8dae318 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-16T05:55:07,801 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/default/testBlockEvictionOnGracefulStop1731736507110/49ee9ebdc7532df078f1ab2a638cddfd/.tmp/family/61d72c4def6246ff80a656f7c8dae318 2024-11-16T05:55:07,803 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/default/testBlockEvictionOnGracefulStop1731736507110/49ee9ebdc7532df078f1ab2a638cddfd/.tmp/family/61d72c4def6246ff80a656f7c8dae318 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/default/testBlockEvictionOnGracefulStop1731736507110/49ee9ebdc7532df078f1ab2a638cddfd/family/61d72c4def6246ff80a656f7c8dae318 2024-11-16T05:55:07,805 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/default/testBlockEvictionOnGracefulStop1731736507110/49ee9ebdc7532df078f1ab2a638cddfd/family/61d72c4def6246ff80a656f7c8dae318, delay=905 ms 2024-11-16T05:55:07,806 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/default/testBlockEvictionOnGracefulStop1731736507110/49ee9ebdc7532df078f1ab2a638cddfd/family/61d72c4def6246ff80a656f7c8dae318, entries=2, sequenceid=6, filesize=4.9 K 2024-11-16T05:55:07,806 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 49ee9ebdc7532df078f1ab2a638cddfd in 26ms, sequenceid=6, compaction requested=false 2024-11-16T05:55:07,806 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1731736507110' 2024-11-16T05:55:07,807 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 49ee9ebdc7532df078f1ab2a638cddfd: 2024-11-16T05:55:07,809 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T05:55:08,710 DEBUG [hfile-prefetch-1731736505064 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61d72c4def6246ff80a656f7c8dae318_0. 2024-11-16T05:55:08,711 DEBUG [hfile-prefetch-1731736505064 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 61d72c4def6246ff80a656f7c8dae318_117. 2024-11-16T05:55:08,711 DEBUG [hfile-prefetch-1731736505064 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/default/testBlockEvictionOnGracefulStop1731736507110/49ee9ebdc7532df078f1ab2a638cddfd/family/61d72c4def6246ff80a656f7c8dae318, with total blocks 2, and data blocks 1 2024-11-16T05:55:08,712 DEBUG [hfile-prefetch-1731736505064 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-16T05:55:08,712 DEBUG [hfile-prefetch-1731736505064 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/default/testBlockEvictionOnGracefulStop1731736507110/49ee9ebdc7532df078f1ab2a638cddfd/family/61d72c4def6246ff80a656f7c8dae318, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-16T05:55:08,712 DEBUG [hfile-prefetch-1731736505064 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/default/testBlockEvictionOnGracefulStop1731736507110/49ee9ebdc7532df078f1ab2a638cddfd/family/61d72c4def6246ff80a656f7c8dae318. Will sleep 100ms and try the verification again. 2024-11-16T05:55:08,751 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-16T05:55:08,752 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-16T05:55:08,752 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-16T05:55:08,752 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-16T05:55:08,755 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-16T05:55:08,755 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731736508752 2024-11-16T05:55:08,755 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-16T05:55:08,808 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;3456ee6a3164:40829,5,FailOnTimeoutGroup] 2024-11-16T05:55:08,808 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '3456ee6a3164,40829,1731736505829' ***** 2024-11-16T05:55:08,808 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-11-16T05:55:08,808 INFO [RS:1;3456ee6a3164:40829 {}] bucket.BucketCache(1738): Disabling cache 2024-11-16T05:55:08,809 INFO [RS:1;3456ee6a3164:40829 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0f004a72-2e32-a29e-dcac-b939763ff566/bucket.persistence 2024-11-16T05:55:08,809 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-16T05:55:08,809 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-16T05:55:08,809 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-16T05:55:08,809 INFO [RS:1;3456ee6a3164:40829 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-16T05:55:08,809 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-16T05:55:08,812 DEBUG [hfile-prefetch-1731736505064 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/default/testBlockEvictionOnGracefulStop1731736507110/49ee9ebdc7532df078f1ab2a638cddfd/family/61d72c4def6246ff80a656f7c8dae318, with total blocks 2, and data blocks 1 2024-11-16T05:55:08,812 DEBUG [hfile-prefetch-1731736505064 {}] bucket.BucketCache(2285): iterating over 1 entries in the backing map 2024-11-16T05:55:08,812 DEBUG [hfile-prefetch-1731736505064 {}] bucket.BucketCache(2293): found block for file 61d72c4def6246ff80a656f7c8dae318 in the backing map. Acquiring read lock for offset 0 2024-11-16T05:55:08,812 DEBUG [hfile-prefetch-1731736505064 {}] bucket.BucketCache(2305): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/default/testBlockEvictionOnGracefulStop1731736507110/49ee9ebdc7532df078f1ab2a638cddfd/family/61d72c4def6246ff80a656f7c8dae318 has now been fully cached. 2024-11-16T05:55:08,812 DEBUG [hfile-prefetch-1731736505064 {}] hfile.PrefetchExecutor(121): Prefetch completed for 61d72c4def6246ff80a656f7c8dae318 2024-11-16T05:55:08,813 INFO [regionserver/3456ee6a3164:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T05:55:08,813 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T05:55:08,819 DEBUG [RS:1;3456ee6a3164:40829 {}] bucket.BucketCache(1487): Thread RS:1;3456ee6a3164:40829 started persisting bucket cache to file 2024-11-16T05:55:08,819 DEBUG [RS:1;3456ee6a3164:40829 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-16T05:55:08,819 DEBUG [RS:1;3456ee6a3164:40829 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-11-16T05:55:08,826 DEBUG [RS:1;3456ee6a3164:40829 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-16T05:55:08,826 DEBUG [RS:1;3456ee6a3164:40829 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1731736508819 2024-11-16T05:55:08,826 DEBUG [RS:1;3456ee6a3164:40829 {}] bucket.BucketCache(1509): Thread RS:1;3456ee6a3164:40829 finished persisting bucket cache to file, renaming 2024-11-16T05:55:08,826 INFO [RS:1;3456ee6a3164:40829 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T05:55:08,827 INFO [RS:1;3456ee6a3164:40829 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-16T05:55:08,827 INFO [RS:1;3456ee6a3164:40829 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-16T05:55:08,827 INFO [RS:1;3456ee6a3164:40829 {}] regionserver.HRegionServer(3091): Received CLOSE for 49ee9ebdc7532df078f1ab2a638cddfd 2024-11-16T05:55:08,827 INFO [RS:1;3456ee6a3164:40829 {}] regionserver.HRegionServer(959): stopping server 3456ee6a3164,40829,1731736505829 2024-11-16T05:55:08,827 INFO [RS:1;3456ee6a3164:40829 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T05:55:08,827 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731736507110,,1731736507125.49ee9ebdc7532df078f1ab2a638cddfd. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-16T05:55:08,827 INFO [RS:1;3456ee6a3164:40829 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;3456ee6a3164:40829. 2024-11-16T05:55:08,827 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 49ee9ebdc7532df078f1ab2a638cddfd, disabling compactions & flushes 2024-11-16T05:55:08,828 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731736507110,,1731736507125.49ee9ebdc7532df078f1ab2a638cddfd. 2024-11-16T05:55:08,828 DEBUG [RS:1;3456ee6a3164:40829 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T05:55:08,828 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731736507110,,1731736507125.49ee9ebdc7532df078f1ab2a638cddfd. 2024-11-16T05:55:08,828 DEBUG [RS:1;3456ee6a3164:40829 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:55:08,828 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731736507110,,1731736507125.49ee9ebdc7532df078f1ab2a638cddfd. after waiting 0 ms 2024-11-16T05:55:08,828 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731736507110,,1731736507125.49ee9ebdc7532df078f1ab2a638cddfd. 2024-11-16T05:55:08,828 INFO [RS:1;3456ee6a3164:40829 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-16T05:55:08,828 INFO [RS:1;3456ee6a3164:40829 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-16T05:55:08,828 INFO [RS:1;3456ee6a3164:40829 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-16T05:55:08,828 INFO [RS:1;3456ee6a3164:40829 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-16T05:55:08,831 INFO [RS:1;3456ee6a3164:40829 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-16T05:55:08,831 DEBUG [RS:1;3456ee6a3164:40829 {}] regionserver.HRegionServer(1325): Online Regions={49ee9ebdc7532df078f1ab2a638cddfd=testBlockEvictionOnGracefulStop1731736507110,,1731736507125.49ee9ebdc7532df078f1ab2a638cddfd., 1588230740=hbase:meta,,1.1588230740} 2024-11-16T05:55:08,831 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-16T05:55:08,831 DEBUG [RS:1;3456ee6a3164:40829 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 49ee9ebdc7532df078f1ab2a638cddfd 2024-11-16T05:55:08,831 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T05:55:08,831 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T05:55:08,831 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T05:55:08,832 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T05:55:08,832 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T05:55:08,832 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-11-16T05:55:08,847 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/default/testBlockEvictionOnGracefulStop1731736507110/49ee9ebdc7532df078f1ab2a638cddfd/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-16T05:55:08,848 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731736507110,,1731736507125.49ee9ebdc7532df078f1ab2a638cddfd. 2024-11-16T05:55:08,848 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 49ee9ebdc7532df078f1ab2a638cddfd: Waiting for close lock at 1731736508827Running coprocessor pre-close hooks at 1731736508827Disabling compacts and flushes for region at 1731736508827Disabling writes for close at 1731736508828 (+1 ms)Writing region close event to WAL at 1731736508833 (+5 ms)Running coprocessor post-close hooks at 1731736508848 (+15 ms)Closed at 1731736508848 2024-11-16T05:55:08,848 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731736507110,,1731736507125.49ee9ebdc7532df078f1ab2a638cddfd. 2024-11-16T05:55:08,856 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 4e340834ec51402597511bbde6cae2da 2024-11-16T05:55:08,876 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/.tmp/info/4e340834ec51402597511bbde6cae2da is 209, key is testBlockEvictionOnGracefulStop1731736507110,,1731736507125.49ee9ebdc7532df078f1ab2a638cddfd./info:regioninfo/1731736507581/Put/seqid=0 2024-11-16T05:55:08,877 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/.tmp/info/4e340834ec51402597511bbde6cae2da 2024-11-16T05:55:08,889 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f4986689c31b4d2f86fdfcad6e66ed3b 2024-11-16T05:55:08,908 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/.tmp/ns/f4986689c31b4d2f86fdfcad6e66ed3b is 43, key is default/ns:d/1731736506894/Put/seqid=0 2024-11-16T05:55:08,909 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/.tmp/ns/f4986689c31b4d2f86fdfcad6e66ed3b 2024-11-16T05:55:08,920 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 04b863d000354eae86ebf75db8813dfe 2024-11-16T05:55:08,935 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/.tmp/table/04b863d000354eae86ebf75db8813dfe is 80, key is testBlockEvictionOnGracefulStop1731736507110/table:state/1731736507590/Put/seqid=0 2024-11-16T05:55:08,936 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/.tmp/table/04b863d000354eae86ebf75db8813dfe 2024-11-16T05:55:08,939 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/.tmp/info/4e340834ec51402597511bbde6cae2da as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/info/4e340834ec51402597511bbde6cae2da 2024-11-16T05:55:08,941 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/info/4e340834ec51402597511bbde6cae2da, delay=935 ms 2024-11-16T05:55:08,943 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/info/4e340834ec51402597511bbde6cae2da, entries=10, sequenceid=11, filesize=7.0 K 2024-11-16T05:55:08,943 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/.tmp/ns/f4986689c31b4d2f86fdfcad6e66ed3b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/ns/f4986689c31b4d2f86fdfcad6e66ed3b 2024-11-16T05:55:08,945 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/ns/f4986689c31b4d2f86fdfcad6e66ed3b, delay=965 ms 2024-11-16T05:55:08,947 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/ns/f4986689c31b4d2f86fdfcad6e66ed3b, entries=2, sequenceid=11, filesize=5.0 K 2024-11-16T05:55:08,947 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/.tmp/table/04b863d000354eae86ebf75db8813dfe as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/table/04b863d000354eae86ebf75db8813dfe 2024-11-16T05:55:08,949 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/table/04b863d000354eae86ebf75db8813dfe, delay=916 ms 2024-11-16T05:55:08,951 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/table/04b863d000354eae86ebf75db8813dfe, entries=2, sequenceid=11, filesize=5.3 K 2024-11-16T05:55:08,952 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 120ms, sequenceid=11, compaction requested=false 2024-11-16T05:55:08,953 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/info/4e340834ec51402597511bbde6cae2da 2024-11-16T05:55:08,954 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/ns/f4986689c31b4d2f86fdfcad6e66ed3b 2024-11-16T05:55:08,955 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/table/04b863d000354eae86ebf75db8813dfe 2024-11-16T05:55:08,960 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-16T05:55:08,960 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-16T05:55:08,960 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T05:55:08,960 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731736508831Running coprocessor pre-close hooks at 1731736508831Disabling compacts and flushes for region at 1731736508831Disabling writes for close at 1731736508832 (+1 ms)Obtaining lock to block concurrent updates at 1731736508832Preparing flush snapshotting stores in 1588230740 at 1731736508832Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1731736508832Flushing stores of hbase:meta,,1.1588230740 at 1731736508833 (+1 ms)Flushing 1588230740/info: creating writer at 1731736508833Flushing 1588230740/info: appending metadata at 1731736508876 (+43 ms)Flushing 1588230740/info: closing flushed file at 1731736508876Flushing 1588230740/ns: creating writer at 1731736508880 (+4 ms)Flushing 1588230740/ns: appending metadata at 1731736508907 (+27 ms)Flushing 1588230740/ns: closing flushed file at 1731736508907Flushing 1588230740/table: creating writer at 1731736508911 (+4 ms)Flushing 1588230740/table: appending metadata at 1731736508935 (+24 ms)Flushing 1588230740/table: closing flushed file at 1731736508935Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e63430f: reopening flushed file at 1731736508939 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@12f1ef9b: reopening flushed file at 1731736508943 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1df0267c: reopening flushed file at 1731736508947 (+4 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 120ms, sequenceid=11, compaction requested=false at 1731736508952 (+5 ms)Writing region close event to WAL at 1731736508955 (+3 ms)Running coprocessor post-close hooks at 1731736508960 (+5 ms)Closed at 1731736508960 2024-11-16T05:55:08,961 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-16T05:55:09,031 INFO [RS:1;3456ee6a3164:40829 {}] regionserver.HRegionServer(976): stopping server 3456ee6a3164,40829,1731736505829; all regions closed. 2024-11-16T05:55:09,034 DEBUG [RS:1;3456ee6a3164:40829 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/oldWALs 2024-11-16T05:55:09,034 INFO [RS:1;3456ee6a3164:40829 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3456ee6a3164%2C40829%2C1731736505829.meta:.meta(num 1731736506829) 2024-11-16T05:55:09,036 DEBUG [RS:1;3456ee6a3164:40829 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/oldWALs 2024-11-16T05:55:09,036 INFO [RS:1;3456ee6a3164:40829 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3456ee6a3164%2C40829%2C1731736505829:(num 1731736506456) 2024-11-16T05:55:09,036 DEBUG [RS:1;3456ee6a3164:40829 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:55:09,036 INFO [RS:1;3456ee6a3164:40829 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T05:55:09,036 INFO [RS:1;3456ee6a3164:40829 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T05:55:09,036 INFO [RS:1;3456ee6a3164:40829 {}] hbase.ChoreService(370): Chore service for: regionserver/3456ee6a3164:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-16T05:55:09,036 INFO [RS:1;3456ee6a3164:40829 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T05:55:09,037 INFO [RS:1;3456ee6a3164:40829 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40829 2024-11-16T05:55:09,038 INFO [regionserver/3456ee6a3164:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T05:55:09,041 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40829-0x1004719697f0002, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/3456ee6a3164,40829,1731736505829 2024-11-16T05:55:09,041 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44763-0x1004719697f0000, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T05:55:09,041 INFO [RS:1;3456ee6a3164:40829 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T05:55:09,043 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [3456ee6a3164,40829,1731736505829] 2024-11-16T05:55:09,045 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/3456ee6a3164,40829,1731736505829 already deleted, retry=false 2024-11-16T05:55:09,046 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 3456ee6a3164,40829,1731736505829 on 3456ee6a3164,44763,1731736505647 2024-11-16T05:55:09,050 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 3456ee6a3164,40829,1731736505829, splitWal=true, meta=true 2024-11-16T05:55:09,053 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 3456ee6a3164,40829,1731736505829 (carryingMeta=true) 3456ee6a3164,40829,1731736505829/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@37456607[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-16T05:55:09,054 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 3456ee6a3164,40829,1731736505829, splitWal=true, meta=true 2024-11-16T05:55:09,055 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 3456ee6a3164,40829,1731736505829, splitWal=true, meta=true, isMeta: true 2024-11-16T05:55:09,055 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/WALs/3456ee6a3164,40829,1731736505829-splitting 2024-11-16T05:55:09,056 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/WALs/3456ee6a3164,40829,1731736505829-splitting dir is empty, no logs to split. 2024-11-16T05:55:09,056 INFO [PEWorker-1 {}] master.SplitWALManager(105): 3456ee6a3164,40829,1731736505829 WAL count=0, meta=true 2024-11-16T05:55:09,058 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/WALs/3456ee6a3164,40829,1731736505829-splitting dir is empty, no logs to split. 2024-11-16T05:55:09,058 INFO [PEWorker-1 {}] master.SplitWALManager(105): 3456ee6a3164,40829,1731736505829 WAL count=0, meta=true 2024-11-16T05:55:09,058 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 3456ee6a3164,40829,1731736505829 WAL splitting is done? wals=0, meta=true 2024-11-16T05:55:09,060 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-16T05:55:09,062 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T05:55:09,063 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-16T05:55:09,143 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40829-0x1004719697f0002, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T05:55:09,143 INFO [RS:1;3456ee6a3164:40829 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T05:55:09,143 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40829-0x1004719697f0002, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T05:55:09,143 INFO [RS:1;3456ee6a3164:40829 {}] regionserver.HRegionServer(1031): Exiting; stopping=3456ee6a3164,40829,1731736505829; zookeeper connection closed. 2024-11-16T05:55:09,143 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3b812280 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3b812280 2024-11-16T05:55:09,213 DEBUG [3456ee6a3164:44763 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-16T05:55:09,214 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=3456ee6a3164,44729,1731736505725 2024-11-16T05:55:09,215 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3456ee6a3164,44729,1731736505725, state=OPENING 2024-11-16T05:55:09,217 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44729-0x1004719697f0001, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:55:09,217 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44763-0x1004719697f0000, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:55:09,217 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T05:55:09,217 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:55:09,217 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:55:09,217 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,44729,1731736505725}] 2024-11-16T05:55:09,333 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/3456ee6a3164:0 server-side Connection retries=45 2024-11-16T05:55:09,333 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T05:55:09,334 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T05:55:09,334 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T05:55:09,334 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T05:55:09,334 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T05:55:09,334 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T05:55:09,334 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T05:55:09,335 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43599 2024-11-16T05:55:09,338 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43599 connecting to ZooKeeper ensemble=127.0.0.1:53959 2024-11-16T05:55:09,339 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T05:55:09,339 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T05:55:09,354 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:435990x0, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T05:55:09,354 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:435990x0, quorum=127.0.0.1:53959, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-16T05:55:09,354 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-16T05:55:09,354 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43599-0x1004719697f0003 connected 2024-11-16T05:55:09,355 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T05:55:09,355 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0f004a72-2e32-a29e-dcac-b939763ff566/bucket.cache 2024-11-16T05:55:09,355 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-16T05:55:09,355 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-16T05:55:09,355 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-16T05:55:09,372 INFO [Thread-207 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-16T05:55:09,373 INFO [Thread-207 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-16T05:55:09,374 INFO [Thread-207 {}] bucket.BucketCache(1705): Retrieved 1 of chunks with blockCount = 1. 2024-11-16T05:55:09,387 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T05:55:09,388 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0f004a72-2e32-a29e-dcac-b939763ff566/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0f004a72-2e32-a29e-dcac-b939763ff566/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-16T05:55:09,395 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50153, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T05:55:09,405 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T05:55:09,411 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:43599-0x1004719697f0003, quorum=127.0.0.1:53959, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-16T05:55:09,413 DEBUG [Thread-207 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-16T05:55:09,413 DEBUG [Thread-207 {}] bucket.BucketCache(1584): Cached File Entry:<61d72c4def6246ff80a656f7c8dae318,<49ee9ebdc7532df078f1ab2a638cddfd,158>> 2024-11-16T05:55:09,413 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43599-0x1004719697f0003, quorum=127.0.0.1:53959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T05:55:09,414 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-16T05:55:09,414 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T05:55:09,414 INFO [Thread-207 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-16T05:55:09,415 INFO [Thread-207 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 1 2024-11-16T05:55:09,415 INFO [Thread-207 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0f004a72-2e32-a29e-dcac-b939763ff566/bucket.persistence is complete. 2024-11-16T05:55:09,415 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-16T05:55:09,416 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=3456ee6a3164%2C44729%2C1731736505725.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/WALs/3456ee6a3164,44729,1731736505725, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/oldWALs, maxLogs=55 2024-11-16T05:55:09,482 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43599 2024-11-16T05:55:09,485 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43599 2024-11-16T05:55:09,509 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43599 2024-11-16T05:55:09,510 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/WALs/3456ee6a3164,44729,1731736505725/3456ee6a3164%2C44729%2C1731736505725.meta.1731736509416.meta 2024-11-16T05:55:09,529 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-16T05:55:09,529 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-16T05:55:09,530 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-16T05:55:09,530 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-16T05:55:09,530 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-16T05:55:09,530 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-16T05:55:09,531 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:55:09,531 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-16T05:55:09,531 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-16T05:55:09,558 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-16T05:55:09,559 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T05:55:09,559 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:55:09,568 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43599 2024-11-16T05:55:09,589 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43599 2024-11-16T05:55:09,590 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/info/4e340834ec51402597511bbde6cae2da, delay=927 ms 2024-11-16T05:55:09,594 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/info/4e340834ec51402597511bbde6cae2da 2024-11-16T05:55:09,594 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:55:09,594 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-16T05:55:09,595 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T05:55:09,595 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:55:09,605 INFO [RS:2;3456ee6a3164:43599 {}] regionserver.HRegionServer(746): ClusterId : 8b126dd9-c05c-4c77-be92-82dc96bb59cc 2024-11-16T05:55:09,605 DEBUG [RS:2;3456ee6a3164:43599 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T05:55:09,608 DEBUG [RS:2;3456ee6a3164:43599 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T05:55:09,608 DEBUG [RS:2;3456ee6a3164:43599 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T05:55:09,610 DEBUG [RS:2;3456ee6a3164:43599 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T05:55:09,611 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/ns/f4986689c31b4d2f86fdfcad6e66ed3b, delay=949 ms 2024-11-16T05:55:09,611 DEBUG [RS:2;3456ee6a3164:43599 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ebfb7a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3456ee6a3164/172.17.0.2:0 2024-11-16T05:55:09,614 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/ns/f4986689c31b4d2f86fdfcad6e66ed3b 2024-11-16T05:55:09,614 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:55:09,615 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-16T05:55:09,615 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T05:55:09,616 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:55:09,616 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:55:09,616 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-16T05:55:09,617 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T05:55:09,617 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:55:09,638 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/table/04b863d000354eae86ebf75db8813dfe, delay=900 ms 2024-11-16T05:55:09,639 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/table/04b863d000354eae86ebf75db8813dfe 2024-11-16T05:55:09,639 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:55:09,640 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T05:55:09,640 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740 2024-11-16T05:55:09,641 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740 2024-11-16T05:55:09,642 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T05:55:09,642 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T05:55:09,643 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T05:55:09,644 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T05:55:09,645 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64362057, jitterRate=-0.0409306138753891}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T05:55:09,645 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-16T05:55:09,646 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731736509531Writing region info on filesystem at 1731736509531Initializing all the Stores at 1731736509531Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731736509531Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731736509557 (+26 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736509557Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731736509558 (+1 ms)Cleaning up temporary data from old regions at 1731736509642 (+84 ms)Running coprocessor post-open hooks at 1731736509645 (+3 ms)Region opened successfully at 1731736509646 (+1 ms) 2024-11-16T05:55:09,648 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1731736509380 2024-11-16T05:55:09,651 DEBUG [RS:2;3456ee6a3164:43599 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;3456ee6a3164:43599 2024-11-16T05:55:09,652 INFO [RS:2;3456ee6a3164:43599 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T05:55:09,652 INFO [RS:2;3456ee6a3164:43599 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T05:55:09,652 DEBUG [RS:2;3456ee6a3164:43599 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T05:55:09,653 INFO [RS:2;3456ee6a3164:43599 {}] regionserver.HRegionServer(2659): reportForDuty to master=3456ee6a3164,44763,1731736505647 with port=43599, startcode=1731736509333 2024-11-16T05:55:09,653 DEBUG [RS:2;3456ee6a3164:43599 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T05:55:09,654 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=3456ee6a3164,44729,1731736505725 2024-11-16T05:55:09,655 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3456ee6a3164,44729,1731736505725, state=OPEN 2024-11-16T05:55:09,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44729-0x1004719697f0001, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:55:09,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44763-0x1004719697f0000, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:55:09,658 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:55:09,658 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:55:09,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43599-0x1004719697f0003, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:55:09,658 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:55:09,658 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=3456ee6a3164,44729,1731736505725 2024-11-16T05:55:09,661 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-16T05:55:09,661 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-16T05:55:09,667 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-16T05:55:09,667 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 604 msec 2024-11-16T05:55:09,668 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 3456ee6a3164,40829,1731736505829 had 2 regions 2024-11-16T05:55:09,670 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 3456ee6a3164,40829,1731736505829, splitWal=true, meta=true, isMeta: false 2024-11-16T05:55:09,670 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/WALs/3456ee6a3164,40829,1731736505829-splitting dir is empty, no logs to split. 2024-11-16T05:55:09,671 INFO [PEWorker-1 {}] master.SplitWALManager(105): 3456ee6a3164,40829,1731736505829 WAL count=0, meta=false 2024-11-16T05:55:09,672 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-16T05:55:09,672 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,44729,1731736505725 in 442 msec 2024-11-16T05:55:09,673 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/WALs/3456ee6a3164,40829,1731736505829-splitting dir is empty, no logs to split. 2024-11-16T05:55:09,673 INFO [PEWorker-1 {}] master.SplitWALManager(105): 3456ee6a3164,40829,1731736505829 WAL count=0, meta=false 2024-11-16T05:55:09,673 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 3456ee6a3164,40829,1731736505829 WAL splitting is done? wals=0, meta=false 2024-11-16T05:55:09,675 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731736507110, region=49ee9ebdc7532df078f1ab2a638cddfd, ASSIGN}] 2024-11-16T05:55:09,676 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731736507110, region=49ee9ebdc7532df078f1ab2a638cddfd, ASSIGN 2024-11-16T05:55:09,677 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731736507110, region=49ee9ebdc7532df078f1ab2a638cddfd, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-16T05:55:09,680 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35613, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T05:55:09,681 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44763 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 3456ee6a3164,43599,1731736509333 2024-11-16T05:55:09,681 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44763 {}] master.ServerManager(517): Registering regionserver=3456ee6a3164,43599,1731736509333 2024-11-16T05:55:09,685 DEBUG [RS:2;3456ee6a3164:43599 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926 2024-11-16T05:55:09,685 DEBUG [RS:2;3456ee6a3164:43599 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-16T05:55:09,685 DEBUG [RS:2;3456ee6a3164:43599 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T05:55:09,687 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44763-0x1004719697f0000, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T05:55:09,688 DEBUG [RS:2;3456ee6a3164:43599 {}] zookeeper.ZKUtil(111): regionserver:43599-0x1004719697f0003, quorum=127.0.0.1:53959, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/3456ee6a3164,43599,1731736509333 2024-11-16T05:55:09,688 WARN [RS:2;3456ee6a3164:43599 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T05:55:09,688 INFO [RS:2;3456ee6a3164:43599 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T05:55:09,688 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [3456ee6a3164,43599,1731736509333] 2024-11-16T05:55:09,688 DEBUG [RS:2;3456ee6a3164:43599 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/WALs/3456ee6a3164,43599,1731736509333 2024-11-16T05:55:09,712 INFO [RS:2;3456ee6a3164:43599 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T05:55:09,733 INFO [RS:2;3456ee6a3164:43599 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T05:55:09,745 INFO [RS:2;3456ee6a3164:43599 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T05:55:09,745 INFO [RS:2;3456ee6a3164:43599 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:09,756 INFO [RS:2;3456ee6a3164:43599 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T05:55:09,757 INFO [RS:2;3456ee6a3164:43599 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T05:55:09,757 INFO [RS:2;3456ee6a3164:43599 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:09,757 DEBUG [RS:2;3456ee6a3164:43599 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:09,758 DEBUG [RS:2;3456ee6a3164:43599 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:09,758 DEBUG [RS:2;3456ee6a3164:43599 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:09,758 DEBUG [RS:2;3456ee6a3164:43599 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:09,758 DEBUG [RS:2;3456ee6a3164:43599 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:09,758 DEBUG [RS:2;3456ee6a3164:43599 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0, corePoolSize=2, maxPoolSize=2 2024-11-16T05:55:09,758 DEBUG [RS:2;3456ee6a3164:43599 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:09,758 DEBUG [RS:2;3456ee6a3164:43599 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:09,758 DEBUG [RS:2;3456ee6a3164:43599 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:09,758 DEBUG [RS:2;3456ee6a3164:43599 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:09,758 DEBUG [RS:2;3456ee6a3164:43599 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:09,759 DEBUG [RS:2;3456ee6a3164:43599 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:55:09,759 DEBUG [RS:2;3456ee6a3164:43599 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/3456ee6a3164:0, corePoolSize=3, maxPoolSize=3 2024-11-16T05:55:09,759 DEBUG [RS:2;3456ee6a3164:43599 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0, corePoolSize=3, maxPoolSize=3 2024-11-16T05:55:09,768 INFO [RS:2;3456ee6a3164:43599 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:09,768 INFO [RS:2;3456ee6a3164:43599 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:09,768 INFO [RS:2;3456ee6a3164:43599 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:09,769 INFO [RS:2;3456ee6a3164:43599 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:09,769 INFO [RS:2;3456ee6a3164:43599 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:09,769 INFO [RS:2;3456ee6a3164:43599 {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,43599,1731736509333-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T05:55:09,786 INFO [RS:2;3456ee6a3164:43599 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T05:55:09,786 INFO [RS:2;3456ee6a3164:43599 {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,43599,1731736509333-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:09,786 INFO [RS:2;3456ee6a3164:43599 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:09,786 INFO [RS:2;3456ee6a3164:43599 {}] regionserver.Replication(171): 3456ee6a3164,43599,1731736509333 started 2024-11-16T05:55:09,806 INFO [RS:2;3456ee6a3164:43599 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T05:55:09,806 INFO [RS:2;3456ee6a3164:43599 {}] regionserver.HRegionServer(1482): Serving as 3456ee6a3164,43599,1731736509333, RpcServer on 3456ee6a3164/172.17.0.2:43599, sessionid=0x1004719697f0003 2024-11-16T05:55:09,806 DEBUG [RS:2;3456ee6a3164:43599 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T05:55:09,806 DEBUG [RS:2;3456ee6a3164:43599 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 3456ee6a3164,43599,1731736509333 2024-11-16T05:55:09,806 DEBUG [RS:2;3456ee6a3164:43599 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3456ee6a3164,43599,1731736509333' 2024-11-16T05:55:09,806 DEBUG [RS:2;3456ee6a3164:43599 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T05:55:09,807 DEBUG [RS:2;3456ee6a3164:43599 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T05:55:09,808 DEBUG [RS:2;3456ee6a3164:43599 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T05:55:09,808 DEBUG [RS:2;3456ee6a3164:43599 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T05:55:09,808 DEBUG [RS:2;3456ee6a3164:43599 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 3456ee6a3164,43599,1731736509333 2024-11-16T05:55:09,808 DEBUG [RS:2;3456ee6a3164:43599 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3456ee6a3164,43599,1731736509333' 2024-11-16T05:55:09,808 DEBUG [RS:2;3456ee6a3164:43599 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T05:55:09,809 DEBUG [RS:2;3456ee6a3164:43599 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T05:55:09,809 DEBUG [RS:2;3456ee6a3164:43599 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T05:55:09,809 INFO [RS:2;3456ee6a3164:43599 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T05:55:09,809 INFO [RS:2;3456ee6a3164:43599 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T05:55:09,828 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=49ee9ebdc7532df078f1ab2a638cddfd, regionState=OPENING, regionLocation=3456ee6a3164,44729,1731736505725 2024-11-16T05:55:09,830 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3456ee6a3164:40829 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:40829 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:55:09,832 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,40829,1731736505829, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,40829,1731736505829, seqNum=-1, error=java.net.ConnectException: Call to address=3456ee6a3164:40829 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:40829 2024-11-16T05:55:09,833 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,40829,1731736505829, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T05:55:09,833 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,40829,1731736505829, seqNum=-1 from cache 2024-11-16T05:55:09,834 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.FailedServers(52): Added failed server with address 3456ee6a3164:40829 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:40829 2024-11-16T05:55:09,910 INFO [RS:2;3456ee6a3164:43599 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T05:55:09,911 INFO [RS:2;3456ee6a3164:43599 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=3456ee6a3164%2C43599%2C1731736509333, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/WALs/3456ee6a3164,43599,1731736509333, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/oldWALs, maxLogs=55 2024-11-16T05:55:09,933 INFO [RS:2;3456ee6a3164:43599 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/WALs/3456ee6a3164,43599,1731736509333/3456ee6a3164%2C43599%2C1731736509333.1731736509911 2024-11-16T05:55:09,934 DEBUG [RS:2;3456ee6a3164:43599 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-16T05:55:09,941 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:55:09,942 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44729,1731736505725, seqNum=-1] 2024-11-16T05:55:09,942 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:55:09,945 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33013, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:55:09,948 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731736507110, region=49ee9ebdc7532df078f1ab2a638cddfd, ASSIGN because future has completed 2024-11-16T05:55:09,949 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 49ee9ebdc7532df078f1ab2a638cddfd, server=3456ee6a3164,44729,1731736505725}] 2024-11-16T05:55:09,964 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-16T05:55:09,964 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-16T05:55:09,966 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-16T05:55:09,966 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-16T05:55:09,966 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-16T05:55:09,966 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-16T05:55:10,104 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731736507110,,1731736507125.49ee9ebdc7532df078f1ab2a638cddfd. 2024-11-16T05:55:10,105 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7752): Opening region: {ENCODED => 49ee9ebdc7532df078f1ab2a638cddfd, NAME => 'testBlockEvictionOnGracefulStop1731736507110,,1731736507125.49ee9ebdc7532df078f1ab2a638cddfd.', STARTKEY => '', ENDKEY => ''} 2024-11-16T05:55:10,105 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731736507110 49ee9ebdc7532df078f1ab2a638cddfd 2024-11-16T05:55:10,105 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731736507110,,1731736507125.49ee9ebdc7532df078f1ab2a638cddfd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:55:10,106 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7794): checking encryption for 49ee9ebdc7532df078f1ab2a638cddfd 2024-11-16T05:55:10,106 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7797): checking classloading for 49ee9ebdc7532df078f1ab2a638cddfd 2024-11-16T05:55:10,106 INFO [StoreOpener-49ee9ebdc7532df078f1ab2a638cddfd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 49ee9ebdc7532df078f1ab2a638cddfd 2024-11-16T05:55:10,107 INFO [StoreOpener-49ee9ebdc7532df078f1ab2a638cddfd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 49ee9ebdc7532df078f1ab2a638cddfd columnFamilyName family 2024-11-16T05:55:10,108 DEBUG [StoreOpener-49ee9ebdc7532df078f1ab2a638cddfd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:55:10,110 DEBUG [StoreFileOpener-49ee9ebdc7532df078f1ab2a638cddfd-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/default/testBlockEvictionOnGracefulStop1731736507110/49ee9ebdc7532df078f1ab2a638cddfd/family/61d72c4def6246ff80a656f7c8dae318, delay=977 ms 2024-11-16T05:55:10,111 DEBUG [StoreOpener-49ee9ebdc7532df078f1ab2a638cddfd-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/default/testBlockEvictionOnGracefulStop1731736507110/49ee9ebdc7532df078f1ab2a638cddfd/family/61d72c4def6246ff80a656f7c8dae318 2024-11-16T05:55:10,111 INFO [StoreOpener-49ee9ebdc7532df078f1ab2a638cddfd-1 {}] regionserver.HStore(327): Store=49ee9ebdc7532df078f1ab2a638cddfd/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:55:10,111 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1038): replaying wal for 49ee9ebdc7532df078f1ab2a638cddfd 2024-11-16T05:55:10,112 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/default/testBlockEvictionOnGracefulStop1731736507110/49ee9ebdc7532df078f1ab2a638cddfd 2024-11-16T05:55:10,113 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/default/testBlockEvictionOnGracefulStop1731736507110/49ee9ebdc7532df078f1ab2a638cddfd 2024-11-16T05:55:10,113 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1048): stopping wal replay for 49ee9ebdc7532df078f1ab2a638cddfd 2024-11-16T05:55:10,113 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1060): Cleaning up temporary data for 49ee9ebdc7532df078f1ab2a638cddfd 2024-11-16T05:55:10,114 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): writing seq id for 49ee9ebdc7532df078f1ab2a638cddfd 2024-11-16T05:55:10,115 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1114): Opened 49ee9ebdc7532df078f1ab2a638cddfd; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66781583, jitterRate=-0.004876866936683655}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T05:55:10,115 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 49ee9ebdc7532df078f1ab2a638cddfd 2024-11-16T05:55:10,116 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1006): Region open journal for 49ee9ebdc7532df078f1ab2a638cddfd: Running coprocessor pre-open hook at 1731736510106Writing region info on filesystem at 1731736510106Initializing all the Stores at 1731736510106Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736510106Cleaning up temporary data from old regions at 1731736510113 (+7 ms)Running coprocessor post-open hooks at 1731736510115 (+2 ms)Region opened successfully at 1731736510116 (+1 ms) 2024-11-16T05:55:10,117 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731736507110,,1731736507125.49ee9ebdc7532df078f1ab2a638cddfd., pid=11, masterSystemTime=1731736510102 2024-11-16T05:55:10,120 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731736507110,,1731736507125.49ee9ebdc7532df078f1ab2a638cddfd. 2024-11-16T05:55:10,120 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731736507110,,1731736507125.49ee9ebdc7532df078f1ab2a638cddfd. 2024-11-16T05:55:10,121 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=49ee9ebdc7532df078f1ab2a638cddfd, regionState=OPEN, openSeqNum=10, regionLocation=3456ee6a3164,44729,1731736505725 2024-11-16T05:55:10,125 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 49ee9ebdc7532df078f1ab2a638cddfd, server=3456ee6a3164,44729,1731736505725 because future has completed 2024-11-16T05:55:10,127 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44763 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=3456ee6a3164,44729,1731736505725, table=testBlockEvictionOnGracefulStop1731736507110, region=49ee9ebdc7532df078f1ab2a638cddfd. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-16T05:55:10,132 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-16T05:55:10,133 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 49ee9ebdc7532df078f1ab2a638cddfd, server=3456ee6a3164,44729,1731736505725 in 180 msec 2024-11-16T05:55:10,138 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7 2024-11-16T05:55:10,138 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(291): removed crashed server 3456ee6a3164,40829,1731736505829 after splitting done 2024-11-16T05:55:10,138 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731736507110, region=49ee9ebdc7532df078f1ab2a638cddfd, ASSIGN in 457 msec 2024-11-16T05:55:10,141 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 3456ee6a3164,40829,1731736505829, splitWal=true, meta=true in 1.0910 sec 2024-11-16T05:55:10,306 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-16T05:55:10,307 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T05:55:10,307 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T05:55:10,307 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:55:10,307 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:55:10,308 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:55:10,308 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-16T05:55:10,308 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=433876673, stopped=false 2024-11-16T05:55:10,308 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=3456ee6a3164,44763,1731736505647 2024-11-16T05:55:10,313 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44729-0x1004719697f0001, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T05:55:10,313 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44763-0x1004719697f0000, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T05:55:10,313 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43599-0x1004719697f0003, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T05:55:10,313 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44763-0x1004719697f0000, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:55:10,313 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:55:10,314 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T05:55:10,314 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T05:55:10,314 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43599-0x1004719697f0003, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:55:10,314 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:55:10,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44729-0x1004719697f0001, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:55:10,315 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '3456ee6a3164,44729,1731736505725' ***** 2024-11-16T05:55:10,315 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-16T05:55:10,315 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '3456ee6a3164,43599,1731736509333' ***** 2024-11-16T05:55:10,315 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-16T05:55:10,315 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44729-0x1004719697f0001, quorum=127.0.0.1:53959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T05:55:10,316 INFO [RS:2;3456ee6a3164:43599 {}] bucket.BucketCache(1738): Disabling cache 2024-11-16T05:55:10,316 INFO [RS:2;3456ee6a3164:43599 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0f004a72-2e32-a29e-dcac-b939763ff566/bucket.persistence 2024-11-16T05:55:10,316 INFO [RS:0;3456ee6a3164:44729 {}] bucket.BucketCache(1738): Disabling cache 2024-11-16T05:55:10,316 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-16T05:55:10,316 INFO [RS:0;3456ee6a3164:44729 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0f004a72-2e32-a29e-dcac-b939763ff566/bucket.persistence 2024-11-16T05:55:10,316 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-16T05:55:10,316 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-16T05:55:10,316 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-16T05:55:10,317 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44763-0x1004719697f0000, quorum=127.0.0.1:53959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T05:55:10,317 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-16T05:55:10,317 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-16T05:55:10,317 INFO [RS:0;3456ee6a3164:44729 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-16T05:55:10,317 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43599-0x1004719697f0003, quorum=127.0.0.1:53959, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T05:55:10,317 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-16T05:55:10,318 INFO [RS:2;3456ee6a3164:43599 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-16T05:55:10,318 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-16T05:55:10,327 DEBUG [RS:0;3456ee6a3164:44729 {}] bucket.BucketCache(1487): Thread RS:0;3456ee6a3164:44729 started persisting bucket cache to file 2024-11-16T05:55:10,328 DEBUG [RS:0;3456ee6a3164:44729 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-16T05:55:10,328 DEBUG [RS:0;3456ee6a3164:44729 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-16T05:55:10,328 DEBUG [RS:2;3456ee6a3164:43599 {}] bucket.BucketCache(1487): Thread RS:2;3456ee6a3164:43599 started persisting bucket cache to file 2024-11-16T05:55:10,328 DEBUG [RS:2;3456ee6a3164:43599 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-16T05:55:10,328 DEBUG [RS:2;3456ee6a3164:43599 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-11-16T05:55:10,330 DEBUG [RS:0;3456ee6a3164:44729 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-16T05:55:10,331 DEBUG [RS:2;3456ee6a3164:43599 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-16T05:55:10,331 DEBUG [RS:0;3456ee6a3164:44729 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731736510327 2024-11-16T05:55:10,331 DEBUG [RS:2;3456ee6a3164:43599 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1731736510328 2024-11-16T05:55:10,331 DEBUG [RS:0;3456ee6a3164:44729 {}] bucket.BucketCache(1509): Thread RS:0;3456ee6a3164:44729 finished persisting bucket cache to file, renaming 2024-11-16T05:55:10,331 DEBUG [RS:2;3456ee6a3164:43599 {}] bucket.BucketCache(1509): Thread RS:2;3456ee6a3164:43599 finished persisting bucket cache to file, renaming 2024-11-16T05:55:10,331 INFO [RS:0;3456ee6a3164:44729 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T05:55:10,331 INFO [RS:0;3456ee6a3164:44729 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-16T05:55:10,331 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T05:55:10,331 INFO [RS:0;3456ee6a3164:44729 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-16T05:55:10,331 INFO [RS:2;3456ee6a3164:43599 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T05:55:10,331 INFO [RS:0;3456ee6a3164:44729 {}] regionserver.HRegionServer(3091): Received CLOSE for 49ee9ebdc7532df078f1ab2a638cddfd 2024-11-16T05:55:10,331 INFO [RS:2;3456ee6a3164:43599 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-16T05:55:10,331 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T05:55:10,331 INFO [RS:2;3456ee6a3164:43599 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-16T05:55:10,331 INFO [RS:2;3456ee6a3164:43599 {}] regionserver.HRegionServer(959): stopping server 3456ee6a3164,43599,1731736509333 2024-11-16T05:55:10,331 INFO [RS:2;3456ee6a3164:43599 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T05:55:10,331 INFO [RS:0;3456ee6a3164:44729 {}] regionserver.HRegionServer(959): stopping server 3456ee6a3164,44729,1731736505725 2024-11-16T05:55:10,331 INFO [RS:0;3456ee6a3164:44729 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T05:55:10,331 INFO [RS:2;3456ee6a3164:43599 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;3456ee6a3164:43599. 2024-11-16T05:55:10,332 INFO [RS:0;3456ee6a3164:44729 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;3456ee6a3164:44729. 2024-11-16T05:55:10,332 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731736507110,,1731736507125.49ee9ebdc7532df078f1ab2a638cddfd. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-16T05:55:10,332 DEBUG [RS:2;3456ee6a3164:43599 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T05:55:10,332 DEBUG [RS:0;3456ee6a3164:44729 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T05:55:10,332 DEBUG [RS:2;3456ee6a3164:43599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:55:10,332 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 49ee9ebdc7532df078f1ab2a638cddfd, disabling compactions & flushes 2024-11-16T05:55:10,332 DEBUG [RS:0;3456ee6a3164:44729 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:55:10,332 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731736507110,,1731736507125.49ee9ebdc7532df078f1ab2a638cddfd. 2024-11-16T05:55:10,332 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731736507110,,1731736507125.49ee9ebdc7532df078f1ab2a638cddfd. 2024-11-16T05:55:10,332 INFO [RS:0;3456ee6a3164:44729 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-16T05:55:10,332 INFO [RS:2;3456ee6a3164:43599 {}] regionserver.HRegionServer(976): stopping server 3456ee6a3164,43599,1731736509333; all regions closed. 2024-11-16T05:55:10,332 INFO [RS:0;3456ee6a3164:44729 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-16T05:55:10,332 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731736507110,,1731736507125.49ee9ebdc7532df078f1ab2a638cddfd. after waiting 0 ms 2024-11-16T05:55:10,332 INFO [RS:0;3456ee6a3164:44729 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-16T05:55:10,332 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731736507110,,1731736507125.49ee9ebdc7532df078f1ab2a638cddfd. 2024-11-16T05:55:10,332 INFO [RS:0;3456ee6a3164:44729 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-16T05:55:10,332 INFO [RS:0;3456ee6a3164:44729 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-16T05:55:10,332 DEBUG [RS:0;3456ee6a3164:44729 {}] regionserver.HRegionServer(1325): Online Regions={49ee9ebdc7532df078f1ab2a638cddfd=testBlockEvictionOnGracefulStop1731736507110,,1731736507125.49ee9ebdc7532df078f1ab2a638cddfd., 1588230740=hbase:meta,,1.1588230740} 2024-11-16T05:55:10,332 DEBUG [RS:0;3456ee6a3164:44729 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 49ee9ebdc7532df078f1ab2a638cddfd 2024-11-16T05:55:10,332 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-16T05:55:10,333 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T05:55:10,333 DEBUG [StoreFileCloser-49ee9ebdc7532df078f1ab2a638cddfd-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/default/testBlockEvictionOnGracefulStop1731736507110/49ee9ebdc7532df078f1ab2a638cddfd/family/61d72c4def6246ff80a656f7c8dae318 2024-11-16T05:55:10,333 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T05:55:10,333 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T05:55:10,333 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T05:55:10,333 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T05:55:10,333 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.21 KB heapSize=2.80 KB 2024-11-16T05:55:10,334 DEBUG [RS:2;3456ee6a3164:43599 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/oldWALs 2024-11-16T05:55:10,334 INFO [RS:2;3456ee6a3164:43599 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3456ee6a3164%2C43599%2C1731736509333:(num 1731736509911) 2024-11-16T05:55:10,334 DEBUG [RS:2;3456ee6a3164:43599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:55:10,334 INFO [RS:2;3456ee6a3164:43599 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T05:55:10,334 INFO [RS:2;3456ee6a3164:43599 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T05:55:10,334 INFO [RS:2;3456ee6a3164:43599 {}] hbase.ChoreService(370): Chore service for: regionserver/3456ee6a3164:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-16T05:55:10,335 INFO [RS:2;3456ee6a3164:43599 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-16T05:55:10,335 INFO [RS:2;3456ee6a3164:43599 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-16T05:55:10,335 INFO [RS:2;3456ee6a3164:43599 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-16T05:55:10,335 INFO [RS:2;3456ee6a3164:43599 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T05:55:10,335 INFO [RS:2;3456ee6a3164:43599 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43599 2024-11-16T05:55:10,335 INFO [regionserver/3456ee6a3164:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T05:55:10,339 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43599-0x1004719697f0003, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/3456ee6a3164,43599,1731736509333 2024-11-16T05:55:10,339 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44763-0x1004719697f0000, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T05:55:10,339 INFO [RS:2;3456ee6a3164:43599 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T05:55:10,340 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/default/testBlockEvictionOnGracefulStop1731736507110/49ee9ebdc7532df078f1ab2a638cddfd/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-16T05:55:10,340 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731736507110,,1731736507125.49ee9ebdc7532df078f1ab2a638cddfd. 2024-11-16T05:55:10,340 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 49ee9ebdc7532df078f1ab2a638cddfd: Waiting for close lock at 1731736510332Running coprocessor pre-close hooks at 1731736510332Disabling compacts and flushes for region at 1731736510332Disabling writes for close at 1731736510332Writing region close event to WAL at 1731736510333 (+1 ms)Running coprocessor post-close hooks at 1731736510340 (+7 ms)Closed at 1731736510340 2024-11-16T05:55:10,340 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731736507110,,1731736507125.49ee9ebdc7532df078f1ab2a638cddfd. 2024-11-16T05:55:10,350 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [3456ee6a3164,43599,1731736509333] 2024-11-16T05:55:10,352 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/3456ee6a3164,43599,1731736509333 already deleted, retry=false 2024-11-16T05:55:10,353 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 3456ee6a3164,43599,1731736509333 expired; onlineServers=1 2024-11-16T05:55:10,353 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 7a48976973ab433294882cdb851f0037 2024-11-16T05:55:10,355 INFO [regionserver/3456ee6a3164:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-16T05:55:10,355 INFO [regionserver/3456ee6a3164:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-16T05:55:10,370 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/.tmp/info/7a48976973ab433294882cdb851f0037 is 209, key is testBlockEvictionOnGracefulStop1731736507110,,1731736507125.49ee9ebdc7532df078f1ab2a638cddfd./info:regioninfo/1731736510120/Put/seqid=0 2024-11-16T05:55:10,371 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.21 KB at sequenceid=19 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/.tmp/info/7a48976973ab433294882cdb851f0037 2024-11-16T05:55:10,374 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/.tmp/info/7a48976973ab433294882cdb851f0037 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/info/7a48976973ab433294882cdb851f0037 2024-11-16T05:55:10,376 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/info/7a48976973ab433294882cdb851f0037, delay=999 ms 2024-11-16T05:55:10,377 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/info/7a48976973ab433294882cdb851f0037, entries=8, sequenceid=19, filesize=6.6 K 2024-11-16T05:55:10,379 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 45ms, sequenceid=19, compaction requested=false 2024-11-16T05:55:10,379 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/info/4e340834ec51402597511bbde6cae2da 2024-11-16T05:55:10,379 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/info/7a48976973ab433294882cdb851f0037 2024-11-16T05:55:10,380 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/ns/f4986689c31b4d2f86fdfcad6e66ed3b 2024-11-16T05:55:10,380 INFO [regionserver/3456ee6a3164:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T05:55:10,380 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/table/04b863d000354eae86ebf75db8813dfe 2024-11-16T05:55:10,386 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/data/hbase/meta/1588230740/recovered.edits/22.seqid, newMaxSeqId=22, maxSeqId=14 2024-11-16T05:55:10,387 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-16T05:55:10,387 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T05:55:10,387 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731736510332Running coprocessor pre-close hooks at 1731736510333 (+1 ms)Disabling compacts and flushes for region at 1731736510333Disabling writes for close at 1731736510333Obtaining lock to block concurrent updates at 1731736510333Preparing flush snapshotting stores in 1588230740 at 1731736510333Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1238, getHeapSize=2800, getOffHeapSize=0, getCellsCount=8 at 1731736510334 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731736510334Flushing 1588230740/info: creating writer at 1731736510334Flushing 1588230740/info: appending metadata at 1731736510369 (+35 ms)Flushing 1588230740/info: closing flushed file at 1731736510369Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d6130fe: reopening flushed file at 1731736510374 (+5 ms)Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 45ms, sequenceid=19, compaction requested=false at 1731736510379 (+5 ms)Writing region close event to WAL at 1731736510381 (+2 ms)Running coprocessor post-close hooks at 1731736510387 (+6 ms)Closed at 1731736510387 2024-11-16T05:55:10,387 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-16T05:55:10,406 INFO [regionserver/3456ee6a3164:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T05:55:10,451 INFO [RS:2;3456ee6a3164:43599 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T05:55:10,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43599-0x1004719697f0003, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T05:55:10,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43599-0x1004719697f0003, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T05:55:10,451 INFO [RS:2;3456ee6a3164:43599 {}] regionserver.HRegionServer(1031): Exiting; stopping=3456ee6a3164,43599,1731736509333; zookeeper connection closed. 2024-11-16T05:55:10,452 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1703e5a0 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1703e5a0 2024-11-16T05:55:10,533 INFO [RS:0;3456ee6a3164:44729 {}] regionserver.HRegionServer(976): stopping server 3456ee6a3164,44729,1731736505725; all regions closed. 2024-11-16T05:55:10,571 DEBUG [RS:0;3456ee6a3164:44729 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/oldWALs 2024-11-16T05:55:10,571 INFO [RS:0;3456ee6a3164:44729 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3456ee6a3164%2C44729%2C1731736505725.meta:.meta(num 1731736509416) 2024-11-16T05:55:10,572 DEBUG [RS:0;3456ee6a3164:44729 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/oldWALs 2024-11-16T05:55:10,573 INFO [RS:0;3456ee6a3164:44729 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3456ee6a3164%2C44729%2C1731736505725:(num 1731736506449) 2024-11-16T05:55:10,573 DEBUG [RS:0;3456ee6a3164:44729 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:55:10,573 INFO [RS:0;3456ee6a3164:44729 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T05:55:10,573 INFO [RS:0;3456ee6a3164:44729 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T05:55:10,573 INFO [RS:0;3456ee6a3164:44729 {}] hbase.ChoreService(370): Chore service for: regionserver/3456ee6a3164:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-16T05:55:10,573 INFO [RS:0;3456ee6a3164:44729 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T05:55:10,573 INFO [regionserver/3456ee6a3164:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T05:55:10,574 INFO [RS:0;3456ee6a3164:44729 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44729 2024-11-16T05:55:10,576 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44729-0x1004719697f0001, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/3456ee6a3164,44729,1731736505725 2024-11-16T05:55:10,576 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44763-0x1004719697f0000, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T05:55:10,576 INFO [RS:0;3456ee6a3164:44729 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T05:55:10,578 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [3456ee6a3164,44729,1731736505725] 2024-11-16T05:55:10,579 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/3456ee6a3164,44729,1731736505725 already deleted, retry=false 2024-11-16T05:55:10,579 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 3456ee6a3164,44729,1731736505725 expired; onlineServers=0 2024-11-16T05:55:10,579 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '3456ee6a3164,44763,1731736505647' ***** 2024-11-16T05:55:10,579 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-16T05:55:10,579 INFO [M:0;3456ee6a3164:44763 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T05:55:10,580 INFO [M:0;3456ee6a3164:44763 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T05:55:10,580 DEBUG [M:0;3456ee6a3164:44763 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-16T05:55:10,580 DEBUG [M:0;3456ee6a3164:44763 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-16T05:55:10,580 DEBUG [master/3456ee6a3164:0:becomeActiveMaster-HFileCleaner.large.0-1731736506174 {}] cleaner.HFileCleaner(306): Exit Thread[master/3456ee6a3164:0:becomeActiveMaster-HFileCleaner.large.0-1731736506174,5,FailOnTimeoutGroup] 2024-11-16T05:55:10,580 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-16T05:55:10,580 INFO [M:0;3456ee6a3164:44763 {}] hbase.ChoreService(370): Chore service for: master/3456ee6a3164:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-16T05:55:10,580 DEBUG [master/3456ee6a3164:0:becomeActiveMaster-HFileCleaner.small.0-1731736506174 {}] cleaner.HFileCleaner(306): Exit Thread[master/3456ee6a3164:0:becomeActiveMaster-HFileCleaner.small.0-1731736506174,5,FailOnTimeoutGroup] 2024-11-16T05:55:10,580 INFO [M:0;3456ee6a3164:44763 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T05:55:10,580 DEBUG [M:0;3456ee6a3164:44763 {}] master.HMaster(1795): Stopping service threads 2024-11-16T05:55:10,580 INFO [M:0;3456ee6a3164:44763 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-16T05:55:10,580 INFO [M:0;3456ee6a3164:44763 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:55:10,581 INFO [M:0;3456ee6a3164:44763 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T05:55:10,581 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-16T05:55:10,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44763-0x1004719697f0000, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T05:55:10,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44763-0x1004719697f0000, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:55:10,582 DEBUG [M:0;3456ee6a3164:44763 {}] zookeeper.ZKUtil(347): master:44763-0x1004719697f0000, quorum=127.0.0.1:53959, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-16T05:55:10,582 WARN [M:0;3456ee6a3164:44763 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-16T05:55:10,582 INFO [M:0;3456ee6a3164:44763 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/.lastflushedseqids 2024-11-16T05:55:10,589 INFO [M:0;3456ee6a3164:44763 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T05:55:10,589 INFO [M:0;3456ee6a3164:44763 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-16T05:55:10,589 DEBUG [M:0;3456ee6a3164:44763 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T05:55:10,589 INFO [M:0;3456ee6a3164:44763 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T05:55:10,589 DEBUG [M:0;3456ee6a3164:44763 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T05:55:10,590 DEBUG [M:0;3456ee6a3164:44763 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T05:55:10,590 DEBUG [M:0;3456ee6a3164:44763 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T05:55:10,590 INFO [M:0;3456ee6a3164:44763 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=39.51 KB heapSize=49.91 KB 2024-11-16T05:55:10,598 DEBUG [M:0;3456ee6a3164:44763 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 77814353671947fc9dc395dff1778fe7 2024-11-16T05:55:10,614 DEBUG [M:0;3456ee6a3164:44763 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/77814353671947fc9dc395dff1778fe7 is 82, key is hbase:meta,,1/info:regioninfo/1731736509653/Put/seqid=0 2024-11-16T05:55:10,615 INFO [M:0;3456ee6a3164:44763 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/77814353671947fc9dc395dff1778fe7 2024-11-16T05:55:10,624 DEBUG [M:0;3456ee6a3164:44763 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 290d071ada4445f0b458dbea058fa895 2024-11-16T05:55:10,644 DEBUG [M:0;3456ee6a3164:44763 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/290d071ada4445f0b458dbea058fa895 is 783, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731736507595/Put/seqid=0 2024-11-16T05:55:10,645 INFO [M:0;3456ee6a3164:44763 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.28 KB at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/290d071ada4445f0b458dbea058fa895 2024-11-16T05:55:10,653 DEBUG [M:0;3456ee6a3164:44763 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 375bc1a9ae794a209abb5ab6bbf67808 2024-11-16T05:55:10,673 DEBUG [M:0;3456ee6a3164:44763 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/375bc1a9ae794a209abb5ab6bbf67808 is 69, key is 3456ee6a3164,43599,1731736509333/rs:state/1731736509682/Put/seqid=0 2024-11-16T05:55:10,674 INFO [M:0;3456ee6a3164:44763 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/375bc1a9ae794a209abb5ab6bbf67808 2024-11-16T05:55:10,677 INFO [M:0;3456ee6a3164:44763 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 375bc1a9ae794a209abb5ab6bbf67808 2024-11-16T05:55:10,677 DEBUG [M:0;3456ee6a3164:44763 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/77814353671947fc9dc395dff1778fe7 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/77814353671947fc9dc395dff1778fe7 2024-11-16T05:55:10,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44729-0x1004719697f0001, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T05:55:10,678 INFO [RS:0;3456ee6a3164:44729 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T05:55:10,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44729-0x1004719697f0001, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T05:55:10,678 INFO [RS:0;3456ee6a3164:44729 {}] regionserver.HRegionServer(1031): Exiting; stopping=3456ee6a3164,44729,1731736505725; zookeeper connection closed. 2024-11-16T05:55:10,679 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6745f688 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6745f688 2024-11-16T05:55:10,679 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-16T05:55:10,680 INFO [M:0;3456ee6a3164:44763 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/77814353671947fc9dc395dff1778fe7, entries=14, sequenceid=102, filesize=5.9 K 2024-11-16T05:55:10,681 DEBUG [M:0;3456ee6a3164:44763 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/290d071ada4445f0b458dbea058fa895 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/290d071ada4445f0b458dbea058fa895 2024-11-16T05:55:10,683 INFO [M:0;3456ee6a3164:44763 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/290d071ada4445f0b458dbea058fa895, entries=11, sequenceid=102, filesize=6.5 K 2024-11-16T05:55:10,684 DEBUG [M:0;3456ee6a3164:44763 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/375bc1a9ae794a209abb5ab6bbf67808 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/375bc1a9ae794a209abb5ab6bbf67808 2024-11-16T05:55:10,686 INFO [M:0;3456ee6a3164:44763 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 375bc1a9ae794a209abb5ab6bbf67808 2024-11-16T05:55:10,687 INFO [M:0;3456ee6a3164:44763 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/71a6bdad-505b-c6c4-b5a2-2a10691d0763/71e9a8fa-b75d-c8f0-785f-f153c965b926/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/375bc1a9ae794a209abb5ab6bbf67808, entries=3, sequenceid=102, filesize=5.3 K 2024-11-16T05:55:10,688 INFO [M:0;3456ee6a3164:44763 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.51 KB/40460, heapSize ~49.62 KB/50808, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 97ms, sequenceid=102, compaction requested=false 2024-11-16T05:55:10,691 INFO [M:0;3456ee6a3164:44763 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T05:55:10,691 DEBUG [M:0;3456ee6a3164:44763 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731736510589Disabling compacts and flushes for region at 1731736510589Disabling writes for close at 1731736510590 (+1 ms)Obtaining lock to block concurrent updates at 1731736510590Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731736510590Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=40460, getHeapSize=51048, getOffHeapSize=0, getCellsCount=127 at 1731736510590Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731736510590Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731736510591 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731736510613 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731736510613Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731736510617 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731736510644 (+27 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731736510644Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731736510648 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731736510672 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731736510672Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e15e006: reopening flushed file at 1731736510677 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@32596dbd: reopening flushed file at 1731736510680 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d3a87b0: reopening flushed file at 1731736510684 (+4 ms)Finished flush of dataSize ~39.51 KB/40460, heapSize ~49.62 KB/50808, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 97ms, sequenceid=102, compaction requested=false at 1731736510688 (+4 ms)Writing region close event to WAL at 1731736510691 (+3 ms)Closed at 1731736510691 2024-11-16T05:55:10,693 INFO [M:0;3456ee6a3164:44763 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-16T05:55:10,693 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T05:55:10,693 INFO [M:0;3456ee6a3164:44763 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44763 2024-11-16T05:55:10,693 INFO [M:0;3456ee6a3164:44763 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T05:55:10,796 INFO [M:0;3456ee6a3164:44763 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T05:55:10,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44763-0x1004719697f0000, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T05:55:10,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44763-0x1004719697f0000, quorum=127.0.0.1:53959, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T05:55:10,798 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-16T05:55:10,799 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-16T05:55:10,799 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T05:55:10,806 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=71 (was 49) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=282 (was 280) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3721 (was 4242)